aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/models
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-02-01 11:18:50 +0100
committerChocobozzz <me@florianbigard.com>2021-02-01 11:23:12 +0100
commit7448551fe5c111fcc7b7448ad864fb15a5c8d87f (patch)
tree47725af0db72dd75a69fbf765444254969ee7ac7 /server/models
parent33c7131be5883d1b25c49adbcf5750b63905a368 (diff)
downloadPeerTube-7448551fe5c111fcc7b7448ad864fb15a5c8d87f.tar.gz
PeerTube-7448551fe5c111fcc7b7448ad864fb15a5c8d87f.tar.zst
PeerTube-7448551fe5c111fcc7b7448ad864fb15a5c8d87f.zip
Fix redundancy with HLS only files
Diffstat (limited to 'server/models')
-rw-r--r--server/models/redundancy/video-redundancy.ts99
1 files changed, 54 insertions, 45 deletions
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index c536c288b..d3b839cfe 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -1,5 +1,5 @@
1import { sample } from 'lodash' 1import { sample } from 'lodash'
2import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize' 2import { col, FindOptions, fn, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
3import { 3import {
4 AllowNull, 4 AllowNull,
5 BeforeDestroy, 5 BeforeDestroy,
@@ -15,7 +15,7 @@ import {
15 UpdatedAt 15 UpdatedAt
16} from 'sequelize-typescript' 16} from 'sequelize-typescript'
17import { getServerActor } from '@server/models/application/application' 17import { getServerActor } from '@server/models/application/application'
18import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models' 18import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model' 19import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
20import { 20import {
21 FileRedundancyInformation, 21 FileRedundancyInformation,
@@ -36,6 +36,7 @@ import { VideoModel } from '../video/video'
36import { VideoChannelModel } from '../video/video-channel' 36import { VideoChannelModel } from '../video/video-channel'
37import { VideoFileModel } from '../video/video-file' 37import { VideoFileModel } from '../video/video-file'
38import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' 38import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
39import { forEachSeries } from 'async'
39 40
40export enum ScopeNames { 41export enum ScopeNames {
41 WITH_VIDEO = 'WITH_VIDEO' 42 WITH_VIDEO = 'WITH_VIDEO'
@@ -261,6 +262,8 @@ export class VideoRedundancyModel extends Model {
261 } 262 }
262 263
263 static async findMostViewToDuplicate (randomizedFactor: number) { 264 static async findMostViewToDuplicate (randomizedFactor: number) {
265 const peertubeActor = await getServerActor()
266
264 // On VideoModel! 267 // On VideoModel!
265 const query = { 268 const query = {
266 attributes: [ 'id', 'views' ], 269 attributes: [ 'id', 'views' ],
@@ -268,10 +271,10 @@ export class VideoRedundancyModel extends Model {
268 order: getVideoSort('-views'), 271 order: getVideoSort('-views'),
269 where: { 272 where: {
270 privacy: VideoPrivacy.PUBLIC, 273 privacy: VideoPrivacy.PUBLIC,
271 isLive: false 274 isLive: false,
275 ...this.buildVideoIdsForDuplication(peertubeActor)
272 }, 276 },
273 include: [ 277 include: [
274 await VideoRedundancyModel.buildVideoFileForDuplication(),
275 VideoRedundancyModel.buildServerRedundancyInclude() 278 VideoRedundancyModel.buildServerRedundancyInclude()
276 ] 279 ]
277 } 280 }
@@ -280,6 +283,8 @@ export class VideoRedundancyModel extends Model {
280 } 283 }
281 284
282 static async findTrendingToDuplicate (randomizedFactor: number) { 285 static async findTrendingToDuplicate (randomizedFactor: number) {
286 const peertubeActor = await getServerActor()
287
283 // On VideoModel! 288 // On VideoModel!
284 const query = { 289 const query = {
285 attributes: [ 'id', 'views' ], 290 attributes: [ 'id', 'views' ],
@@ -289,10 +294,10 @@ export class VideoRedundancyModel extends Model {
289 order: getVideoSort('-trending'), 294 order: getVideoSort('-trending'),
290 where: { 295 where: {
291 privacy: VideoPrivacy.PUBLIC, 296 privacy: VideoPrivacy.PUBLIC,
292 isLive: false 297 isLive: false,
298 ...this.buildVideoIdsForDuplication(peertubeActor)
293 }, 299 },
294 include: [ 300 include: [
295 await VideoRedundancyModel.buildVideoFileForDuplication(),
296 VideoRedundancyModel.buildServerRedundancyInclude(), 301 VideoRedundancyModel.buildServerRedundancyInclude(),
297 302
298 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) 303 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
@@ -303,6 +308,8 @@ export class VideoRedundancyModel extends Model {
303 } 308 }
304 309
305 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { 310 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
311 const peertubeActor = await getServerActor()
312
306 // On VideoModel! 313 // On VideoModel!
307 const query = { 314 const query = {
308 attributes: [ 'id', 'publishedAt' ], 315 attributes: [ 'id', 'publishedAt' ],
@@ -313,10 +320,10 @@ export class VideoRedundancyModel extends Model {
313 isLive: false, 320 isLive: false,
314 views: { 321 views: {
315 [Op.gte]: minViews 322 [Op.gte]: minViews
316 } 323 },
324 ...this.buildVideoIdsForDuplication(peertubeActor)
317 }, 325 },
318 include: [ 326 include: [
319 await VideoRedundancyModel.buildVideoFileForDuplication(),
320 VideoRedundancyModel.buildServerRedundancyInclude() 327 VideoRedundancyModel.buildServerRedundancyInclude()
321 ] 328 ]
322 } 329 }
@@ -573,32 +580,35 @@ export class VideoRedundancyModel extends Model {
573 static async getStats (strategy: VideoRedundancyStrategyWithManual) { 580 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
574 const actor = await getServerActor() 581 const actor = await getServerActor()
575 582
576 const query: FindOptions = { 583 const sql = `WITH "tmp" AS ` +
577 raw: true, 584 `(` +
578 attributes: [ 585 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
579 [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ], 586 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
580 [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ], 587 `FROM "videoRedundancy" AS "videoRedundancy" ` +
581 [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ] 588 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
582 ], 589 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
583 where: { 590 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
584 strategy, 591 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
585 actorId: actor.id 592 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
586 }, 593 `), ` +
587 include: [ 594 `"videoIds" AS (` +
588 { 595 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
589 attributes: [], 596 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
590 model: VideoFileModel, 597 `) ` +
591 required: true 598 `SELECT ` +
592 } 599 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
593 ] 600 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
594 } 601 `COUNT(*) AS "totalVideoFiles" ` +
595 602 `FROM "tmp"`
596 return VideoRedundancyModel.findOne(query) 603
597 .then((r: any) => ({ 604 return VideoRedundancyModel.sequelize.query<any>(sql, {
598 totalUsed: parseAggregateResult(r.totalUsed), 605 replacements: { strategy, actorId: actor.id },
599 totalVideos: r.totalVideos, 606 type: QueryTypes.SELECT
600 totalVideoFiles: r.totalVideoFiles 607 }).then(([ row ]) => ({
601 })) 608 totalUsed: parseAggregateResult(row.totalUsed),
609 totalVideos: row.totalVideos,
610 totalVideoFiles: row.totalVideoFiles
611 }))
602 } 612 }
603 613
604 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { 614 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
@@ -692,23 +702,22 @@ export class VideoRedundancyModel extends Model {
692 } 702 }
693 703
694 // Don't include video files we already duplicated 704 // Don't include video files we already duplicated
695 private static async buildVideoFileForDuplication () { 705 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
696 const actor = await getServerActor()
697
698 const notIn = literal( 706 const notIn = literal(
699 '(' + 707 '(' +
700 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + 708 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
709 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
710 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
711 `UNION ` +
712 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
713 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
714 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
701 ')' 715 ')'
702 ) 716 )
703 717
704 return { 718 return {
705 attributes: [], 719 id: {
706 model: VideoFileModel, 720 [Op.notIn]: notIn
707 required: true,
708 where: {
709 id: {
710 [Op.notIn]: notIn
711 }
712 } 721 }
713 } 722 }
714 } 723 }