aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--server/models/redundancy/video-redundancy.ts99
-rw-r--r--server/tests/api/redundancy/redundancy.ts90
-rw-r--r--shared/extra-utils/videos/videos.ts2
3 files changed, 132 insertions, 59 deletions
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index c536c288b..d3b839cfe 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -1,5 +1,5 @@
1import { sample } from 'lodash' 1import { sample } from 'lodash'
2import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize' 2import { col, FindOptions, fn, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
3import { 3import {
4 AllowNull, 4 AllowNull,
5 BeforeDestroy, 5 BeforeDestroy,
@@ -15,7 +15,7 @@ import {
15 UpdatedAt 15 UpdatedAt
16} from 'sequelize-typescript' 16} from 'sequelize-typescript'
17import { getServerActor } from '@server/models/application/application' 17import { getServerActor } from '@server/models/application/application'
18import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models' 18import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model' 19import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
20import { 20import {
21 FileRedundancyInformation, 21 FileRedundancyInformation,
@@ -36,6 +36,7 @@ import { VideoModel } from '../video/video'
36import { VideoChannelModel } from '../video/video-channel' 36import { VideoChannelModel } from '../video/video-channel'
37import { VideoFileModel } from '../video/video-file' 37import { VideoFileModel } from '../video/video-file'
38import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' 38import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
39import { forEachSeries } from 'async'
39 40
40export enum ScopeNames { 41export enum ScopeNames {
41 WITH_VIDEO = 'WITH_VIDEO' 42 WITH_VIDEO = 'WITH_VIDEO'
@@ -261,6 +262,8 @@ export class VideoRedundancyModel extends Model {
261 } 262 }
262 263
263 static async findMostViewToDuplicate (randomizedFactor: number) { 264 static async findMostViewToDuplicate (randomizedFactor: number) {
265 const peertubeActor = await getServerActor()
266
264 // On VideoModel! 267 // On VideoModel!
265 const query = { 268 const query = {
266 attributes: [ 'id', 'views' ], 269 attributes: [ 'id', 'views' ],
@@ -268,10 +271,10 @@ export class VideoRedundancyModel extends Model {
268 order: getVideoSort('-views'), 271 order: getVideoSort('-views'),
269 where: { 272 where: {
270 privacy: VideoPrivacy.PUBLIC, 273 privacy: VideoPrivacy.PUBLIC,
271 isLive: false 274 isLive: false,
275 ...this.buildVideoIdsForDuplication(peertubeActor)
272 }, 276 },
273 include: [ 277 include: [
274 await VideoRedundancyModel.buildVideoFileForDuplication(),
275 VideoRedundancyModel.buildServerRedundancyInclude() 278 VideoRedundancyModel.buildServerRedundancyInclude()
276 ] 279 ]
277 } 280 }
@@ -280,6 +283,8 @@ export class VideoRedundancyModel extends Model {
280 } 283 }
281 284
282 static async findTrendingToDuplicate (randomizedFactor: number) { 285 static async findTrendingToDuplicate (randomizedFactor: number) {
286 const peertubeActor = await getServerActor()
287
283 // On VideoModel! 288 // On VideoModel!
284 const query = { 289 const query = {
285 attributes: [ 'id', 'views' ], 290 attributes: [ 'id', 'views' ],
@@ -289,10 +294,10 @@ export class VideoRedundancyModel extends Model {
289 order: getVideoSort('-trending'), 294 order: getVideoSort('-trending'),
290 where: { 295 where: {
291 privacy: VideoPrivacy.PUBLIC, 296 privacy: VideoPrivacy.PUBLIC,
292 isLive: false 297 isLive: false,
298 ...this.buildVideoIdsForDuplication(peertubeActor)
293 }, 299 },
294 include: [ 300 include: [
295 await VideoRedundancyModel.buildVideoFileForDuplication(),
296 VideoRedundancyModel.buildServerRedundancyInclude(), 301 VideoRedundancyModel.buildServerRedundancyInclude(),
297 302
298 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) 303 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
@@ -303,6 +308,8 @@ export class VideoRedundancyModel extends Model {
303 } 308 }
304 309
305 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { 310 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
311 const peertubeActor = await getServerActor()
312
306 // On VideoModel! 313 // On VideoModel!
307 const query = { 314 const query = {
308 attributes: [ 'id', 'publishedAt' ], 315 attributes: [ 'id', 'publishedAt' ],
@@ -313,10 +320,10 @@ export class VideoRedundancyModel extends Model {
313 isLive: false, 320 isLive: false,
314 views: { 321 views: {
315 [Op.gte]: minViews 322 [Op.gte]: minViews
316 } 323 },
324 ...this.buildVideoIdsForDuplication(peertubeActor)
317 }, 325 },
318 include: [ 326 include: [
319 await VideoRedundancyModel.buildVideoFileForDuplication(),
320 VideoRedundancyModel.buildServerRedundancyInclude() 327 VideoRedundancyModel.buildServerRedundancyInclude()
321 ] 328 ]
322 } 329 }
@@ -573,32 +580,35 @@ export class VideoRedundancyModel extends Model {
573 static async getStats (strategy: VideoRedundancyStrategyWithManual) { 580 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
574 const actor = await getServerActor() 581 const actor = await getServerActor()
575 582
576 const query: FindOptions = { 583 const sql = `WITH "tmp" AS ` +
577 raw: true, 584 `(` +
578 attributes: [ 585 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
579 [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ], 586 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
580 [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ], 587 `FROM "videoRedundancy" AS "videoRedundancy" ` +
581 [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ] 588 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
582 ], 589 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
583 where: { 590 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
584 strategy, 591 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
585 actorId: actor.id 592 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
586 }, 593 `), ` +
587 include: [ 594 `"videoIds" AS (` +
588 { 595 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
589 attributes: [], 596 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
590 model: VideoFileModel, 597 `) ` +
591 required: true 598 `SELECT ` +
592 } 599 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
593 ] 600 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
594 } 601 `COUNT(*) AS "totalVideoFiles" ` +
595 602 `FROM "tmp"`
596 return VideoRedundancyModel.findOne(query) 603
597 .then((r: any) => ({ 604 return VideoRedundancyModel.sequelize.query<any>(sql, {
598 totalUsed: parseAggregateResult(r.totalUsed), 605 replacements: { strategy, actorId: actor.id },
599 totalVideos: r.totalVideos, 606 type: QueryTypes.SELECT
600 totalVideoFiles: r.totalVideoFiles 607 }).then(([ row ]) => ({
601 })) 608 totalUsed: parseAggregateResult(row.totalUsed),
609 totalVideos: row.totalVideos,
610 totalVideoFiles: row.totalVideoFiles
611 }))
602 } 612 }
603 613
604 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { 614 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
@@ -692,23 +702,22 @@ export class VideoRedundancyModel extends Model {
692 } 702 }
693 703
694 // Don't include video files we already duplicated 704 // Don't include video files we already duplicated
695 private static async buildVideoFileForDuplication () { 705 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
696 const actor = await getServerActor()
697
698 const notIn = literal( 706 const notIn = literal(
699 '(' + 707 '(' +
700 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + 708 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
709 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
710 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
711 `UNION ` +
712 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
713 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
714 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
701 ')' 715 ')'
702 ) 716 )
703 717
704 return { 718 return {
705 attributes: [], 719 id: {
706 model: VideoFileModel, 720 [Op.notIn]: notIn
707 required: true,
708 where: {
709 id: {
710 [Op.notIn]: notIn
711 }
712 } 721 }
713 } 722 }
714 } 723 }
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 7cfcf70e1..8da0ba72a 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -21,6 +21,8 @@ import {
21 ServerInfo, 21 ServerInfo,
22 setAccessTokensToServers, 22 setAccessTokensToServers,
23 unfollow, 23 unfollow,
24 updateCustomConfig,
25 updateCustomSubConfig,
24 uploadVideo, 26 uploadVideo,
25 viewVideo, 27 viewVideo,
26 wait, 28 wait,
@@ -60,7 +62,7 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
60 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) 62 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
61} 63}
62 64
63async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) { 65async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
64 const strategies: any[] = [] 66 const strategies: any[] = []
65 67
66 if (strategy !== null) { 68 if (strategy !== null) {
@@ -75,6 +77,9 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, add
75 77
76 const config = { 78 const config = {
77 transcoding: { 79 transcoding: {
80 webtorrent: {
81 enabled: withWebtorrent
82 },
78 hls: { 83 hls: {
79 enabled: true 84 enabled: true
80 } 85 }
@@ -253,7 +258,7 @@ async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
253 return stat 258 return stat
254} 259}
255 260
256async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) { 261async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual) {
257 const stat = await checkStatsGlobal(strategy) 262 const stat = await checkStatsGlobal(strategy)
258 263
259 expect(stat.totalUsed).to.be.at.least(1).and.below(409601) 264 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
@@ -261,7 +266,7 @@ async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManu
261 expect(stat.totalVideos).to.equal(1) 266 expect(stat.totalVideos).to.equal(1)
262} 267}
263 268
264async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) { 269async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
265 const stat = await checkStatsGlobal(strategy) 270 const stat = await checkStatsGlobal(strategy)
266 271
267 expect(stat.totalUsed).to.equal(0) 272 expect(stat.totalUsed).to.equal(0)
@@ -313,7 +318,7 @@ describe('Test videos redundancy', function () {
313 it('Should have 1 webseed on the first video', async function () { 318 it('Should have 1 webseed on the first video', async function () {
314 await check1WebSeed() 319 await check1WebSeed()
315 await check0PlaylistRedundancies() 320 await check0PlaylistRedundancies()
316 await checkStatsWith1Webseed(strategy) 321 await checkStatsWithoutRedundancy(strategy)
317 }) 322 })
318 323
319 it('Should enable redundancy on server 1', function () { 324 it('Should enable redundancy on server 1', function () {
@@ -329,7 +334,7 @@ describe('Test videos redundancy', function () {
329 334
330 await check2Webseeds() 335 await check2Webseeds()
331 await check1PlaylistRedundancies() 336 await check1PlaylistRedundancies()
332 await checkStatsWith2Webseed(strategy) 337 await checkStatsWith1Redundancy(strategy)
333 }) 338 })
334 339
335 it('Should undo redundancy on server 1 and remove duplicated videos', async function () { 340 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
@@ -363,7 +368,7 @@ describe('Test videos redundancy', function () {
363 it('Should have 1 webseed on the first video', async function () { 368 it('Should have 1 webseed on the first video', async function () {
364 await check1WebSeed() 369 await check1WebSeed()
365 await check0PlaylistRedundancies() 370 await check0PlaylistRedundancies()
366 await checkStatsWith1Webseed(strategy) 371 await checkStatsWithoutRedundancy(strategy)
367 }) 372 })
368 373
369 it('Should enable redundancy on server 1', function () { 374 it('Should enable redundancy on server 1', function () {
@@ -379,7 +384,7 @@ describe('Test videos redundancy', function () {
379 384
380 await check2Webseeds() 385 await check2Webseeds()
381 await check1PlaylistRedundancies() 386 await check1PlaylistRedundancies()
382 await checkStatsWith2Webseed(strategy) 387 await checkStatsWith1Redundancy(strategy)
383 }) 388 })
384 389
385 it('Should unfollow on server 1 and remove duplicated videos', async function () { 390 it('Should unfollow on server 1 and remove duplicated videos', async function () {
@@ -413,7 +418,7 @@ describe('Test videos redundancy', function () {
413 it('Should have 1 webseed on the first video', async function () { 418 it('Should have 1 webseed on the first video', async function () {
414 await check1WebSeed() 419 await check1WebSeed()
415 await check0PlaylistRedundancies() 420 await check0PlaylistRedundancies()
416 await checkStatsWith1Webseed(strategy) 421 await checkStatsWithoutRedundancy(strategy)
417 }) 422 })
418 423
419 it('Should enable redundancy on server 1', function () { 424 it('Should enable redundancy on server 1', function () {
@@ -429,7 +434,7 @@ describe('Test videos redundancy', function () {
429 434
430 await check1WebSeed() 435 await check1WebSeed()
431 await check0PlaylistRedundancies() 436 await check0PlaylistRedundancies()
432 await checkStatsWith1Webseed(strategy) 437 await checkStatsWithoutRedundancy(strategy)
433 }) 438 })
434 439
435 it('Should view 2 times the first video to have > min_views config', async function () { 440 it('Should view 2 times the first video to have > min_views config', async function () {
@@ -451,7 +456,7 @@ describe('Test videos redundancy', function () {
451 456
452 await check2Webseeds() 457 await check2Webseeds()
453 await check1PlaylistRedundancies() 458 await check1PlaylistRedundancies()
454 await checkStatsWith2Webseed(strategy) 459 await checkStatsWith1Redundancy(strategy)
455 }) 460 })
456 461
457 it('Should remove the video and the redundancy files', async function () { 462 it('Should remove the video and the redundancy files', async function () {
@@ -471,6 +476,65 @@ describe('Test videos redundancy', function () {
471 }) 476 })
472 }) 477 })
473 478
479 describe('With only HLS files', function () {
480 const strategy = 'recently-added'
481
482 before(async function () {
483 this.timeout(120000)
484
485 await flushAndRunServers(strategy, { min_views: 3 }, false)
486 })
487
488 it('Should have 0 playlist redundancy on the first video', async function () {
489 await check1WebSeed()
490 await check0PlaylistRedundancies()
491 })
492
493 it('Should enable redundancy on server 1', function () {
494 return enableRedundancyOnServer1()
495 })
496
497 it('Should still have 0 redundancy on the first video', async function () {
498 this.timeout(80000)
499
500 await waitJobs(servers)
501 await wait(15000)
502 await waitJobs(servers)
503
504 await check0PlaylistRedundancies()
505 await checkStatsWithoutRedundancy(strategy)
506 })
507
508 it('Should have 1 redundancy on the first video', async function () {
509 this.timeout(160000)
510
511 await viewVideo(servers[0].url, video1Server2UUID)
512 await viewVideo(servers[2].url, video1Server2UUID)
513
514 await wait(10000)
515 await waitJobs(servers)
516
517 await waitJobs(servers)
518 await waitUntilLog(servers[0], 'Duplicated ', 1)
519 await waitJobs(servers)
520
521 await check1PlaylistRedundancies()
522 await checkStatsWith1Redundancy(strategy)
523 })
524
525 it('Should remove the video and the redundancy files', async function () {
526 this.timeout(20000)
527
528 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
529
530 await waitJobs(servers)
531
532 for (const server of servers) {
533 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
534 }
535 })
536 })
537
474 describe('With manual strategy', function () { 538 describe('With manual strategy', function () {
475 before(function () { 539 before(function () {
476 this.timeout(120000) 540 this.timeout(120000)
@@ -481,7 +545,7 @@ describe('Test videos redundancy', function () {
481 it('Should have 1 webseed on the first video', async function () { 545 it('Should have 1 webseed on the first video', async function () {
482 await check1WebSeed() 546 await check1WebSeed()
483 await check0PlaylistRedundancies() 547 await check0PlaylistRedundancies()
484 await checkStatsWith1Webseed('manual') 548 await checkStatsWithoutRedundancy('manual')
485 }) 549 })
486 550
487 it('Should create a redundancy on first video', async function () { 551 it('Should create a redundancy on first video', async function () {
@@ -501,7 +565,7 @@ describe('Test videos redundancy', function () {
501 565
502 await check2Webseeds() 566 await check2Webseeds()
503 await check1PlaylistRedundancies() 567 await check1PlaylistRedundancies()
504 await checkStatsWith2Webseed('manual') 568 await checkStatsWith1Redundancy('manual')
505 }) 569 })
506 570
507 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { 571 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
@@ -619,7 +683,7 @@ describe('Test videos redundancy', function () {
619 683
620 await check2Webseeds() 684 await check2Webseeds()
621 await check1PlaylistRedundancies() 685 await check1PlaylistRedundancies()
622 await checkStatsWith2Webseed(strategy) 686 await checkStatsWith1Redundancy(strategy)
623 687
624 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' }) 688 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' })
625 video2Server2UUID = res.body.video.uuid 689 video2Server2UUID = res.body.video.uuid
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts
index 110552c77..f94fa233c 100644
--- a/shared/extra-utils/videos/videos.ts
+++ b/shared/extra-utils/videos/videos.ts
@@ -338,7 +338,7 @@ async function checkVideoFilesWereRemoved (
338 338
339 const files = await readdir(directoryPath) 339 const files = await readdir(directoryPath)
340 for (const file of files) { 340 for (const file of files) {
341 expect(file).to.not.contain(videoUUID) 341 expect(file, `File ${file} should not exist in ${directoryPath}`).to.not.contain(videoUUID)
342 } 342 }
343 } 343 }
344} 344}