server/tests/api/redundancy/redundancy.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { readdir } from 'fs-extra'
   6 import * as magnetUtil from 'magnet-uri'
   7 import { join } from 'path'
   8 import { HttpStatusCode } from '@shared/core-utils'
   9 import {
  10   checkSegmentHash,
  11   checkVideoFilesWereRemoved,
  12   cleanupTests,
  13   doubleFollow,
  14   flushAndRunMultipleServers,
  15   getVideo,
  16   getVideoWithToken,
  17   immutableAssign,
  18   killallServers,
  19   makeGetRequest,
  20   removeVideo,
  21   reRunServer,
  22   root,
  23   ServerInfo,
  24   setAccessTokensToServers,
  25   updateVideo,
  26   uploadVideo,
  27   viewVideo,
  28   wait,
  29   waitJobs,
  30   waitUntilLog
  31 } from '@shared/extra-utils'
  32 import { VideoDetails, VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '@shared/models'
  33
  34 const expect = chai.expect
  35
  36 let servers: ServerInfo[] = []
  37 let video1Server2UUID: string
  38 let video1Server2Id: number
  39
  40 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  41   const parsed = magnetUtil.decode(file.magnetUri)
  42
  43   for (const ws of baseWebseeds) {
  44     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  45     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  46   }
  47
  48   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  49 }
  50
  51 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
  52   const strategies: any[] = []
  53
  54   if (strategy !== null) {
  55     strategies.push(
  56       immutableAssign({
  57         min_lifetime: '1 hour',
  58         strategy: strategy,
  59         size: '400KB'
  60       }, additionalParams)
  61     )
  62   }
  63
  64   const config = {
  65     transcoding: {
  66       webtorrent: {
  67         enabled: withWebtorrent
  68       },
  69       hls: {
  70         enabled: true
  71       }
  72     },
  73     redundancy: {
  74       videos: {
  75         check_interval: '5 seconds',
  76         strategies
  77       }
  78     }
  79   }
  80
  81   servers = await flushAndRunMultipleServers(3, config)
  82
  83   // Get the access tokens
  84   await setAccessTokensToServers(servers)
  85
  86   {
  87     const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
  88     video1Server2UUID = res.body.video.uuid
  89     video1Server2Id = res.body.video.id
  90
  91     await viewVideo(servers[1].url, video1Server2UUID)
  92   }
  93
  94   await waitJobs(servers)
  95
  96   // Server 1 and server 2 follow each other
  97   await doubleFollow(servers[0], servers[1])
  98   // Server 1 and server 3 follow each other
  99   await doubleFollow(servers[0], servers[2])
 100   // Server 2 and server 3 follow each other
 101   await doubleFollow(servers[1], servers[2])
 102
 103   await waitJobs(servers)
 104 }
 105
 106 async function check1WebSeed (videoUUID?: string) {
 107   if (!videoUUID) videoUUID = video1Server2UUID
 108
 109   const webseeds = [
 110     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 111   ]
 112
 113   for (const server of servers) {
 114     // With token to avoid issues with video follow constraints
 115     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 116
 117     const video: VideoDetails = res.body
 118     for (const f of video.files) {
 119       checkMagnetWebseeds(f, webseeds, server)
 120     }
 121   }
 122 }
 123
 124 async function check2Webseeds (videoUUID?: string) {
 125   if (!videoUUID) videoUUID = video1Server2UUID
 126
 127   const webseeds = [
 128     `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
 129     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 130   ]
 131
 132   for (const server of servers) {
 133     const res = await getVideo(server.url, videoUUID)
 134
 135     const video: VideoDetails = res.body
 136
 137     for (const file of video.files) {
 138       checkMagnetWebseeds(file, webseeds, server)
 139
 140       await makeGetRequest({
 141         url: servers[0].url,
 142         statusCodeExpected: HttpStatusCode.OK_200,
 143         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 144         contentType: null
 145       })
 146       await makeGetRequest({
 147         url: servers[1].url,
 148         statusCodeExpected: HttpStatusCode.OK_200,
 149         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 150         contentType: null
 151       })
 152     }
 153   }
 154
 155   const directories = [
 156     'test' + servers[0].internalServerNumber + '/redundancy',
 157     'test' + servers[1].internalServerNumber + '/videos'
 158   ]
 159
 160   for (const directory of directories) {
 161     const files = await readdir(join(root(), directory))
 162     expect(files).to.have.length.at.least(4)
 163
 164     for (const resolution of [ 240, 360, 480, 720 ]) {
 165       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 166     }
 167   }
 168 }
 169
 170 async function check0PlaylistRedundancies (videoUUID?: string) {
 171   if (!videoUUID) videoUUID = video1Server2UUID
 172
 173   for (const server of servers) {
 174     // With token to avoid issues with video follow constraints
 175     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 176     const video: VideoDetails = res.body
 177
 178     expect(video.streamingPlaylists).to.be.an('array')
 179     expect(video.streamingPlaylists).to.have.lengthOf(1)
 180     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 181   }
 182 }
 183
 184 async function check1PlaylistRedundancies (videoUUID?: string) {
 185   if (!videoUUID) videoUUID = video1Server2UUID
 186
 187   for (const server of servers) {
 188     const res = await getVideo(server.url, videoUUID)
 189     const video: VideoDetails = res.body
 190
 191     expect(video.streamingPlaylists).to.have.lengthOf(1)
 192     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 193
 194     const redundancy = video.streamingPlaylists[0].redundancies[0]
 195
 196     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 197   }
 198
 199   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 200   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 201
 202   const res = await getVideo(servers[0].url, videoUUID)
 203   const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
 204
 205   for (const resolution of [ 240, 360, 480, 720 ]) {
 206     await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist })
 207   }
 208
 209   const directories = [
 210     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 211     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 212   ]
 213
 214   for (const directory of directories) {
 215     const files = await readdir(join(root(), directory, videoUUID))
 216     expect(files).to.have.length.at.least(4)
 217
 218     for (const resolution of [ 240, 360, 480, 720 ]) {
 219       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 220
 221       expect(files.find(f => f === filename)).to.not.be.undefined
 222     }
 223   }
 224 }
 225
 226 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
 227   let totalSize: number = null
 228   let statsLength = 1
 229
 230   if (strategy !== 'manual') {
 231     totalSize = 409600
 232     statsLength = 2
 233   }
 234
 235   const data = await servers[0].statsCommand.get()
 236   expect(data.videosRedundancy).to.have.lengthOf(statsLength)
 237
 238   const stat = data.videosRedundancy[0]
 239   expect(stat.strategy).to.equal(strategy)
 240   expect(stat.totalSize).to.equal(totalSize)
 241
 242   return stat
 243 }
 244
 245 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
 246   const stat = await checkStatsGlobal(strategy)
 247
 248   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 249   expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
 250   expect(stat.totalVideos).to.equal(1)
 251 }
 252
 253 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
 254   const stat = await checkStatsGlobal(strategy)
 255
 256   expect(stat.totalUsed).to.equal(0)
 257   expect(stat.totalVideoFiles).to.equal(0)
 258   expect(stat.totalVideos).to.equal(0)
 259 }
 260
 261 async function findServerFollows () {
 262   const body = await servers[0].followsCommand.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
 263   const follows = body.data
 264   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 265   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 266
 267   return { server2, server3 }
 268 }
 269
 270 async function enableRedundancyOnServer1 () {
 271   await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
 272
 273   const { server2, server3 } = await findServerFollows()
 274
 275   expect(server3).to.not.be.undefined
 276   expect(server3.following.hostRedundancyAllowed).to.be.false
 277
 278   expect(server2).to.not.be.undefined
 279   expect(server2.following.hostRedundancyAllowed).to.be.true
 280 }
 281
 282 async function disableRedundancyOnServer1 () {
 283   await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
 284
 285   const { server2, server3 } = await findServerFollows()
 286
 287   expect(server3).to.not.be.undefined
 288   expect(server3.following.hostRedundancyAllowed).to.be.false
 289
 290   expect(server2).to.not.be.undefined
 291   expect(server2.following.hostRedundancyAllowed).to.be.false
 292 }
 293
 294 describe('Test videos redundancy', function () {
 295
 296   describe('With most-views strategy', function () {
 297     const strategy = 'most-views'
 298
 299     before(function () {
 300       this.timeout(120000)
 301
 302       return flushAndRunServers(strategy)
 303     })
 304
 305     it('Should have 1 webseed on the first video', async function () {
 306       await check1WebSeed()
 307       await check0PlaylistRedundancies()
 308       await checkStatsWithoutRedundancy(strategy)
 309     })
 310
 311     it('Should enable redundancy on server 1', function () {
 312       return enableRedundancyOnServer1()
 313     })
 314
 315     it('Should have 2 webseeds on the first video', async function () {
 316       this.timeout(80000)
 317
 318       await waitJobs(servers)
 319       await waitUntilLog(servers[0], 'Duplicated ', 5)
 320       await waitJobs(servers)
 321
 322       await check2Webseeds()
 323       await check1PlaylistRedundancies()
 324       await checkStatsWith1Redundancy(strategy)
 325     })
 326
 327     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 328       this.timeout(80000)
 329
 330       await disableRedundancyOnServer1()
 331
 332       await waitJobs(servers)
 333       await wait(5000)
 334
 335       await check1WebSeed()
 336       await check0PlaylistRedundancies()
 337
 338       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
 339     })
 340
 341     after(async function () {
 342       return cleanupTests(servers)
 343     })
 344   })
 345
 346   describe('With trending strategy', function () {
 347     const strategy = 'trending'
 348
 349     before(function () {
 350       this.timeout(120000)
 351
 352       return flushAndRunServers(strategy)
 353     })
 354
 355     it('Should have 1 webseed on the first video', async function () {
 356       await check1WebSeed()
 357       await check0PlaylistRedundancies()
 358       await checkStatsWithoutRedundancy(strategy)
 359     })
 360
 361     it('Should enable redundancy on server 1', function () {
 362       return enableRedundancyOnServer1()
 363     })
 364
 365     it('Should have 2 webseeds on the first video', async function () {
 366       this.timeout(80000)
 367
 368       await waitJobs(servers)
 369       await waitUntilLog(servers[0], 'Duplicated ', 5)
 370       await waitJobs(servers)
 371
 372       await check2Webseeds()
 373       await check1PlaylistRedundancies()
 374       await checkStatsWith1Redundancy(strategy)
 375     })
 376
 377     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 378       this.timeout(80000)
 379
 380       await servers[0].followsCommand.unfollow({ target: servers[1] })
 381
 382       await waitJobs(servers)
 383       await wait(5000)
 384
 385       await check1WebSeed()
 386       await check0PlaylistRedundancies()
 387
 388       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
 389     })
 390
 391     after(async function () {
 392       await cleanupTests(servers)
 393     })
 394   })
 395
 396   describe('With recently added strategy', function () {
 397     const strategy = 'recently-added'
 398
 399     before(function () {
 400       this.timeout(120000)
 401
 402       return flushAndRunServers(strategy, { min_views: 3 })
 403     })
 404
 405     it('Should have 1 webseed on the first video', async function () {
 406       await check1WebSeed()
 407       await check0PlaylistRedundancies()
 408       await checkStatsWithoutRedundancy(strategy)
 409     })
 410
 411     it('Should enable redundancy on server 1', function () {
 412       return enableRedundancyOnServer1()
 413     })
 414
 415     it('Should still have 1 webseed on the first video', async function () {
 416       this.timeout(80000)
 417
 418       await waitJobs(servers)
 419       await wait(15000)
 420       await waitJobs(servers)
 421
 422       await check1WebSeed()
 423       await check0PlaylistRedundancies()
 424       await checkStatsWithoutRedundancy(strategy)
 425     })
 426
 427     it('Should view 2 times the first video to have > min_views config', async function () {
 428       this.timeout(80000)
 429
 430       await viewVideo(servers[0].url, video1Server2UUID)
 431       await viewVideo(servers[2].url, video1Server2UUID)
 432
 433       await wait(10000)
 434       await waitJobs(servers)
 435     })
 436
 437     it('Should have 2 webseeds on the first video', async function () {
 438       this.timeout(80000)
 439
 440       await waitJobs(servers)
 441       await waitUntilLog(servers[0], 'Duplicated ', 5)
 442       await waitJobs(servers)
 443
 444       await check2Webseeds()
 445       await check1PlaylistRedundancies()
 446       await checkStatsWith1Redundancy(strategy)
 447     })
 448
 449     it('Should remove the video and the redundancy files', async function () {
 450       this.timeout(20000)
 451
 452       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 453
 454       await waitJobs(servers)
 455
 456       for (const server of servers) {
 457         await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
 458       }
 459     })
 460
 461     after(async function () {
 462       await cleanupTests(servers)
 463     })
 464   })
 465
 466   describe('With only HLS files', function () {
 467     const strategy = 'recently-added'
 468
 469     before(async function () {
 470       this.timeout(120000)
 471
 472       await flushAndRunServers(strategy, { min_views: 3 }, false)
 473     })
 474
 475     it('Should have 0 playlist redundancy on the first video', async function () {
 476       await check1WebSeed()
 477       await check0PlaylistRedundancies()
 478     })
 479
 480     it('Should enable redundancy on server 1', function () {
 481       return enableRedundancyOnServer1()
 482     })
 483
 484     it('Should still have 0 redundancy on the first video', async function () {
 485       this.timeout(80000)
 486
 487       await waitJobs(servers)
 488       await wait(15000)
 489       await waitJobs(servers)
 490
 491       await check0PlaylistRedundancies()
 492       await checkStatsWithoutRedundancy(strategy)
 493     })
 494
 495     it('Should have 1 redundancy on the first video', async function () {
 496       this.timeout(160000)
 497
 498       await viewVideo(servers[0].url, video1Server2UUID)
 499       await viewVideo(servers[2].url, video1Server2UUID)
 500
 501       await wait(10000)
 502       await waitJobs(servers)
 503
 504       await waitJobs(servers)
 505       await waitUntilLog(servers[0], 'Duplicated ', 1)
 506       await waitJobs(servers)
 507
 508       await check1PlaylistRedundancies()
 509       await checkStatsWith1Redundancy(strategy, true)
 510     })
 511
 512     it('Should remove the video and the redundancy files', async function () {
 513       this.timeout(20000)
 514
 515       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 516
 517       await waitJobs(servers)
 518
 519       for (const server of servers) {
 520         await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
 521       }
 522     })
 523
 524     after(async function () {
 525       await cleanupTests(servers)
 526     })
 527   })
 528
 529   describe('With manual strategy', function () {
 530     before(function () {
 531       this.timeout(120000)
 532
 533       return flushAndRunServers(null)
 534     })
 535
 536     it('Should have 1 webseed on the first video', async function () {
 537       await check1WebSeed()
 538       await check0PlaylistRedundancies()
 539       await checkStatsWithoutRedundancy('manual')
 540     })
 541
 542     it('Should create a redundancy on first video', async function () {
 543       await servers[0].redundancyCommand.addVideo({ videoId: video1Server2Id })
 544     })
 545
 546     it('Should have 2 webseeds on the first video', async function () {
 547       this.timeout(80000)
 548
 549       await waitJobs(servers)
 550       await waitUntilLog(servers[0], 'Duplicated ', 5)
 551       await waitJobs(servers)
 552
 553       await check2Webseeds()
 554       await check1PlaylistRedundancies()
 555       await checkStatsWith1Redundancy('manual')
 556     })
 557
 558     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
 559       this.timeout(80000)
 560
 561       const body = await servers[0].redundancyCommand.listVideos({ target: 'remote-videos' })
 562
 563       const videos = body.data
 564       expect(videos).to.have.lengthOf(1)
 565
 566       const video = videos[0]
 567
 568       for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
 569         await servers[0].redundancyCommand.removeVideo({ redundancyId: r.id })
 570       }
 571
 572       await waitJobs(servers)
 573       await wait(5000)
 574
 575       await check1WebSeed()
 576       await check0PlaylistRedundancies()
 577
 578       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 579     })
 580
 581     after(async function () {
 582       await cleanupTests(servers)
 583     })
 584   })
 585
 586   describe('Test expiration', function () {
 587     const strategy = 'recently-added'
 588
 589     async function checkContains (servers: ServerInfo[], str: string) {
 590       for (const server of servers) {
 591         const res = await getVideo(server.url, video1Server2UUID)
 592         const video: VideoDetails = res.body
 593
 594         for (const f of video.files) {
 595           expect(f.magnetUri).to.contain(str)
 596         }
 597       }
 598     }
 599
 600     async function checkNotContains (servers: ServerInfo[], str: string) {
 601       for (const server of servers) {
 602         const res = await getVideo(server.url, video1Server2UUID)
 603         const video: VideoDetails = res.body
 604
 605         for (const f of video.files) {
 606           expect(f.magnetUri).to.not.contain(str)
 607         }
 608       }
 609     }
 610
 611     before(async function () {
 612       this.timeout(120000)
 613
 614       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 615
 616       await enableRedundancyOnServer1()
 617     })
 618
 619     it('Should still have 2 webseeds after 10 seconds', async function () {
 620       this.timeout(80000)
 621
 622       await wait(10000)
 623
 624       try {
 625         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 626       } catch {
 627         // Maybe a server deleted a redundancy in the scheduler
 628         await wait(2000)
 629
 630         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 631       }
 632     })
 633
 634     it('Should stop server 1 and expire video redundancy', async function () {
 635       this.timeout(80000)
 636
 637       killallServers([ servers[0] ])
 638
 639       await wait(15000)
 640
 641       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 642     })
 643
 644     after(async function () {
 645       await cleanupTests(servers)
 646     })
 647   })
 648
 649   describe('Test file replacement', function () {
 650     let video2Server2UUID: string
 651     const strategy = 'recently-added'
 652
 653     before(async function () {
 654       this.timeout(120000)
 655
 656       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 657
 658       await enableRedundancyOnServer1()
 659
 660       await waitJobs(servers)
 661       await waitUntilLog(servers[0], 'Duplicated ', 5)
 662       await waitJobs(servers)
 663
 664       await check2Webseeds(video1Server2UUID)
 665       await check1PlaylistRedundancies(video1Server2UUID)
 666       await checkStatsWith1Redundancy(strategy)
 667
 668       const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE })
 669       video2Server2UUID = res.body.video.uuid
 670
 671       // Wait transcoding before federation
 672       await waitJobs(servers)
 673
 674       await updateVideo(servers[1].url, servers[1].accessToken, video2Server2UUID, { privacy: VideoPrivacy.PUBLIC })
 675     })
 676
 677     it('Should cache video 2 webseeds on the first video', async function () {
 678       this.timeout(120000)
 679
 680       await waitJobs(servers)
 681
 682       let checked = false
 683
 684       while (checked === false) {
 685         await wait(1000)
 686
 687         try {
 688           await check1WebSeed(video1Server2UUID)
 689           await check0PlaylistRedundancies(video1Server2UUID)
 690
 691           await check2Webseeds(video2Server2UUID)
 692           await check1PlaylistRedundancies(video2Server2UUID)
 693
 694           checked = true
 695         } catch {
 696           checked = false
 697         }
 698       }
 699     })
 700
 701     it('Should disable strategy and remove redundancies', async function () {
 702       this.timeout(80000)
 703
 704       await waitJobs(servers)
 705
 706       killallServers([ servers[0] ])
 707       await reRunServer(servers[0], {
 708         redundancy: {
 709           videos: {
 710             check_interval: '1 second',
 711             strategies: []
 712           }
 713         }
 714       })
 715
 716       await waitJobs(servers)
 717
 718       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
 719     })
 720
 721     after(async function () {
 722       await cleanupTests(servers)
 723     })
 724   })
 725 })