server/tests/api/redundancy/redundancy.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { readdir } from 'fs-extra'
   6 import magnetUtil from 'magnet-uri'
   7 import { basename, join } from 'path'
   8 import {
   9   checkSegmentHash,
  10   checkVideoFilesWereRemoved,
  11   cleanupTests,
  12   createMultipleServers,
  13   doubleFollow,
  14   killallServers,
  15   makeRawRequest,
  16   PeerTubeServer,
  17   root,
  18   saveVideoInServers,
  19   setAccessTokensToServers,
  20   wait,
  21   waitJobs
  22 } from '@shared/server-commands'
  23 import {
  24   HttpStatusCode,
  25   VideoDetails,
  26   VideoFile,
  27   VideoPrivacy,
  28   VideoRedundancyStrategy,
  29   VideoRedundancyStrategyWithManual
  30 } from '@shared/models'
  31
  32 const expect = chai.expect
  33
  34 let servers: PeerTubeServer[] = []
  35 let video1Server2: VideoDetails
  36
  37 async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) {
  38   const parsed = magnetUtil.decode(file.magnetUri)
  39
  40   for (const ws of baseWebseeds) {
  41     const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`)
  42     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  43   }
  44
  45   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  46
  47   for (const url of parsed.urlList) {
  48     await makeRawRequest(url, HttpStatusCode.OK_200)
  49   }
  50 }
  51
  52 async function createServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
  53   const strategies: any[] = []
  54
  55   if (strategy !== null) {
  56     strategies.push(
  57       {
  58         min_lifetime: '1 hour',
  59         strategy: strategy,
  60         size: '400KB',
  61
  62         ...additionalParams
  63       }
  64     )
  65   }
  66
  67   const config = {
  68     transcoding: {
  69       webtorrent: {
  70         enabled: withWebtorrent
  71       },
  72       hls: {
  73         enabled: true
  74       }
  75     },
  76     redundancy: {
  77       videos: {
  78         check_interval: '5 seconds',
  79         strategies
  80       }
  81     }
  82   }
  83
  84   servers = await createMultipleServers(3, config)
  85
  86   // Get the access tokens
  87   await setAccessTokensToServers(servers)
  88
  89   {
  90     const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } })
  91     video1Server2 = await servers[1].videos.get({ id })
  92
  93     await servers[1].videos.view({ id })
  94   }
  95
  96   await waitJobs(servers)
  97
  98   // Server 1 and server 2 follow each other
  99   await doubleFollow(servers[0], servers[1])
 100   // Server 1 and server 3 follow each other
 101   await doubleFollow(servers[0], servers[2])
 102   // Server 2 and server 3 follow each other
 103   await doubleFollow(servers[1], servers[2])
 104
 105   await waitJobs(servers)
 106 }
 107
 108 async function ensureSameFilenames (videoUUID: string) {
 109   let webtorrentFilenames: string[]
 110   let hlsFilenames: string[]
 111
 112   for (const server of servers) {
 113     const video = await server.videos.getWithToken({ id: videoUUID })
 114
 115     // Ensure we use the same filenames that the origin
 116
 117     const localWebtorrentFilenames = video.files.map(f => basename(f.fileUrl)).sort()
 118     const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort()
 119
 120     if (webtorrentFilenames) expect(webtorrentFilenames).to.deep.equal(localWebtorrentFilenames)
 121     else webtorrentFilenames = localWebtorrentFilenames
 122
 123     if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames)
 124     else hlsFilenames = localHLSFilenames
 125   }
 126
 127   return { webtorrentFilenames, hlsFilenames }
 128 }
 129
 130 async function check1WebSeed (videoUUID?: string) {
 131   if (!videoUUID) videoUUID = video1Server2.uuid
 132
 133   const webseeds = [
 134     `http://localhost:${servers[1].port}/static/webseed/`
 135   ]
 136
 137   for (const server of servers) {
 138     // With token to avoid issues with video follow constraints
 139     const video = await server.videos.getWithToken({ id: videoUUID })
 140
 141     for (const f of video.files) {
 142       await checkMagnetWebseeds(f, webseeds, server)
 143     }
 144   }
 145
 146   await ensureSameFilenames(videoUUID)
 147 }
 148
 149 async function check2Webseeds (videoUUID?: string) {
 150   if (!videoUUID) videoUUID = video1Server2.uuid
 151
 152   const webseeds = [
 153     `http://localhost:${servers[0].port}/static/redundancy/`,
 154     `http://localhost:${servers[1].port}/static/webseed/`
 155   ]
 156
 157   for (const server of servers) {
 158     const video = await server.videos.get({ id: videoUUID })
 159
 160     for (const file of video.files) {
 161       await checkMagnetWebseeds(file, webseeds, server)
 162     }
 163   }
 164
 165   const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
 166
 167   const directories = [
 168     'test' + servers[0].internalServerNumber + '/redundancy',
 169     'test' + servers[1].internalServerNumber + '/videos'
 170   ]
 171
 172   for (const directory of directories) {
 173     const files = await readdir(join(root(), directory))
 174     expect(files).to.have.length.at.least(4)
 175
 176     // Ensure we files exist on disk
 177     expect(files.find(f => webtorrentFilenames.includes(f))).to.exist
 178   }
 179 }
 180
 181 async function check0PlaylistRedundancies (videoUUID?: string) {
 182   if (!videoUUID) videoUUID = video1Server2.uuid
 183
 184   for (const server of servers) {
 185     // With token to avoid issues with video follow constraints
 186     const video = await server.videos.getWithToken({ id: videoUUID })
 187
 188     expect(video.streamingPlaylists).to.be.an('array')
 189     expect(video.streamingPlaylists).to.have.lengthOf(1)
 190     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 191   }
 192
 193   await ensureSameFilenames(videoUUID)
 194 }
 195
 196 async function check1PlaylistRedundancies (videoUUID?: string) {
 197   if (!videoUUID) videoUUID = video1Server2.uuid
 198
 199   for (const server of servers) {
 200     const video = await server.videos.get({ id: videoUUID })
 201
 202     expect(video.streamingPlaylists).to.have.lengthOf(1)
 203     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 204
 205     const redundancy = video.streamingPlaylists[0].redundancies[0]
 206
 207     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 208   }
 209
 210   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID
 211   const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID
 212
 213   const video = await servers[0].videos.get({ id: videoUUID })
 214   const hlsPlaylist = video.streamingPlaylists[0]
 215
 216   for (const resolution of [ 240, 360, 480, 720 ]) {
 217     await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist })
 218   }
 219
 220   const { hlsFilenames } = await ensureSameFilenames(videoUUID)
 221
 222   const directories = [
 223     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 224     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 225   ]
 226
 227   for (const directory of directories) {
 228     const files = await readdir(join(root(), directory, videoUUID))
 229     expect(files).to.have.length.at.least(4)
 230
 231     // Ensure we files exist on disk
 232     expect(files.find(f => hlsFilenames.includes(f))).to.exist
 233   }
 234 }
 235
 236 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
 237   let totalSize: number = null
 238   let statsLength = 1
 239
 240   if (strategy !== 'manual') {
 241     totalSize = 409600
 242     statsLength = 2
 243   }
 244
 245   const data = await servers[0].stats.get()
 246   expect(data.videosRedundancy).to.have.lengthOf(statsLength)
 247
 248   const stat = data.videosRedundancy[0]
 249   expect(stat.strategy).to.equal(strategy)
 250   expect(stat.totalSize).to.equal(totalSize)
 251
 252   return stat
 253 }
 254
 255 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
 256   const stat = await checkStatsGlobal(strategy)
 257
 258   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 259   expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
 260   expect(stat.totalVideos).to.equal(1)
 261 }
 262
 263 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
 264   const stat = await checkStatsGlobal(strategy)
 265
 266   expect(stat.totalUsed).to.equal(0)
 267   expect(stat.totalVideoFiles).to.equal(0)
 268   expect(stat.totalVideos).to.equal(0)
 269 }
 270
 271 async function findServerFollows () {
 272   const body = await servers[0].follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
 273   const follows = body.data
 274   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 275   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 276
 277   return { server2, server3 }
 278 }
 279
 280 async function enableRedundancyOnServer1 () {
 281   await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
 282
 283   const { server2, server3 } = await findServerFollows()
 284
 285   expect(server3).to.not.be.undefined
 286   expect(server3.following.hostRedundancyAllowed).to.be.false
 287
 288   expect(server2).to.not.be.undefined
 289   expect(server2.following.hostRedundancyAllowed).to.be.true
 290 }
 291
 292 async function disableRedundancyOnServer1 () {
 293   await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
 294
 295   const { server2, server3 } = await findServerFollows()
 296
 297   expect(server3).to.not.be.undefined
 298   expect(server3.following.hostRedundancyAllowed).to.be.false
 299
 300   expect(server2).to.not.be.undefined
 301   expect(server2.following.hostRedundancyAllowed).to.be.false
 302 }
 303
 304 describe('Test videos redundancy', function () {
 305
 306   describe('With most-views strategy', function () {
 307     const strategy = 'most-views'
 308
 309     before(function () {
 310       this.timeout(240000)
 311
 312       return createServers(strategy)
 313     })
 314
 315     it('Should have 1 webseed on the first video', async function () {
 316       await check1WebSeed()
 317       await check0PlaylistRedundancies()
 318       await checkStatsWithoutRedundancy(strategy)
 319     })
 320
 321     it('Should enable redundancy on server 1', function () {
 322       return enableRedundancyOnServer1()
 323     })
 324
 325     it('Should have 2 webseeds on the first video', async function () {
 326       this.timeout(80000)
 327
 328       await waitJobs(servers)
 329       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 330       await waitJobs(servers)
 331
 332       await check2Webseeds()
 333       await check1PlaylistRedundancies()
 334       await checkStatsWith1Redundancy(strategy)
 335     })
 336
 337     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 338       this.timeout(80000)
 339
 340       await disableRedundancyOnServer1()
 341
 342       await waitJobs(servers)
 343       await wait(5000)
 344
 345       await check1WebSeed()
 346       await check0PlaylistRedundancies()
 347
 348       await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
 349     })
 350
 351     after(async function () {
 352       return cleanupTests(servers)
 353     })
 354   })
 355
 356   describe('With trending strategy', function () {
 357     const strategy = 'trending'
 358
 359     before(function () {
 360       this.timeout(240000)
 361
 362       return createServers(strategy)
 363     })
 364
 365     it('Should have 1 webseed on the first video', async function () {
 366       await check1WebSeed()
 367       await check0PlaylistRedundancies()
 368       await checkStatsWithoutRedundancy(strategy)
 369     })
 370
 371     it('Should enable redundancy on server 1', function () {
 372       return enableRedundancyOnServer1()
 373     })
 374
 375     it('Should have 2 webseeds on the first video', async function () {
 376       this.timeout(80000)
 377
 378       await waitJobs(servers)
 379       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 380       await waitJobs(servers)
 381
 382       await check2Webseeds()
 383       await check1PlaylistRedundancies()
 384       await checkStatsWith1Redundancy(strategy)
 385     })
 386
 387     it('Should unfollow server 3 and keep duplicated videos', async function () {
 388       this.timeout(80000)
 389
 390       await servers[0].follows.unfollow({ target: servers[2] })
 391
 392       await waitJobs(servers)
 393       await wait(5000)
 394
 395       await check2Webseeds()
 396       await check1PlaylistRedundancies()
 397       await checkStatsWith1Redundancy(strategy)
 398     })
 399
 400     it('Should unfollow server 2 and remove duplicated videos', async function () {
 401       this.timeout(80000)
 402
 403       await servers[0].follows.unfollow({ target: servers[1] })
 404
 405       await waitJobs(servers)
 406       await wait(5000)
 407
 408       await check1WebSeed()
 409       await check0PlaylistRedundancies()
 410
 411       await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
 412     })
 413
 414     after(async function () {
 415       await cleanupTests(servers)
 416     })
 417   })
 418
 419   describe('With recently added strategy', function () {
 420     const strategy = 'recently-added'
 421
 422     before(function () {
 423       this.timeout(240000)
 424
 425       return createServers(strategy, { min_views: 3 })
 426     })
 427
 428     it('Should have 1 webseed on the first video', async function () {
 429       await check1WebSeed()
 430       await check0PlaylistRedundancies()
 431       await checkStatsWithoutRedundancy(strategy)
 432     })
 433
 434     it('Should enable redundancy on server 1', function () {
 435       return enableRedundancyOnServer1()
 436     })
 437
 438     it('Should still have 1 webseed on the first video', async function () {
 439       this.timeout(80000)
 440
 441       await waitJobs(servers)
 442       await wait(15000)
 443       await waitJobs(servers)
 444
 445       await check1WebSeed()
 446       await check0PlaylistRedundancies()
 447       await checkStatsWithoutRedundancy(strategy)
 448     })
 449
 450     it('Should view 2 times the first video to have > min_views config', async function () {
 451       this.timeout(80000)
 452
 453       await servers[0].videos.view({ id: video1Server2.uuid })
 454       await servers[2].videos.view({ id: video1Server2.uuid })
 455
 456       await wait(10000)
 457       await waitJobs(servers)
 458     })
 459
 460     it('Should have 2 webseeds on the first video', async function () {
 461       this.timeout(80000)
 462
 463       await waitJobs(servers)
 464       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 465       await waitJobs(servers)
 466
 467       await check2Webseeds()
 468       await check1PlaylistRedundancies()
 469       await checkStatsWith1Redundancy(strategy)
 470     })
 471
 472     it('Should remove the video and the redundancy files', async function () {
 473       this.timeout(20000)
 474
 475       await saveVideoInServers(servers, video1Server2.uuid)
 476       await servers[1].videos.remove({ id: video1Server2.uuid })
 477
 478       await waitJobs(servers)
 479
 480       for (const server of servers) {
 481         await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails })
 482       }
 483     })
 484
 485     after(async function () {
 486       await cleanupTests(servers)
 487     })
 488   })
 489
 490   describe('With only HLS files', function () {
 491     const strategy = 'recently-added'
 492
 493     before(async function () {
 494       this.timeout(240000)
 495
 496       await createServers(strategy, { min_views: 3 }, false)
 497     })
 498
 499     it('Should have 0 playlist redundancy on the first video', async function () {
 500       await check1WebSeed()
 501       await check0PlaylistRedundancies()
 502     })
 503
 504     it('Should enable redundancy on server 1', function () {
 505       return enableRedundancyOnServer1()
 506     })
 507
 508     it('Should still have 0 redundancy on the first video', async function () {
 509       this.timeout(80000)
 510
 511       await waitJobs(servers)
 512       await wait(15000)
 513       await waitJobs(servers)
 514
 515       await check0PlaylistRedundancies()
 516       await checkStatsWithoutRedundancy(strategy)
 517     })
 518
 519     it('Should have 1 redundancy on the first video', async function () {
 520       this.timeout(160000)
 521
 522       await servers[0].videos.view({ id: video1Server2.uuid })
 523       await servers[2].videos.view({ id: video1Server2.uuid })
 524
 525       await wait(10000)
 526       await waitJobs(servers)
 527
 528       await waitJobs(servers)
 529       await servers[0].servers.waitUntilLog('Duplicated ', 1)
 530       await waitJobs(servers)
 531
 532       await check1PlaylistRedundancies()
 533       await checkStatsWith1Redundancy(strategy, true)
 534     })
 535
 536     it('Should remove the video and the redundancy files', async function () {
 537       this.timeout(20000)
 538
 539       await saveVideoInServers(servers, video1Server2.uuid)
 540       await servers[1].videos.remove({ id: video1Server2.uuid })
 541
 542       await waitJobs(servers)
 543
 544       for (const server of servers) {
 545         await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails })
 546       }
 547     })
 548
 549     after(async function () {
 550       await cleanupTests(servers)
 551     })
 552   })
 553
 554   describe('With manual strategy', function () {
 555     before(function () {
 556       this.timeout(240000)
 557
 558       return createServers(null)
 559     })
 560
 561     it('Should have 1 webseed on the first video', async function () {
 562       await check1WebSeed()
 563       await check0PlaylistRedundancies()
 564       await checkStatsWithoutRedundancy('manual')
 565     })
 566
 567     it('Should create a redundancy on first video', async function () {
 568       await servers[0].redundancy.addVideo({ videoId: video1Server2.id })
 569     })
 570
 571     it('Should have 2 webseeds on the first video', async function () {
 572       this.timeout(80000)
 573
 574       await waitJobs(servers)
 575       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 576       await waitJobs(servers)
 577
 578       await check2Webseeds()
 579       await check1PlaylistRedundancies()
 580       await checkStatsWith1Redundancy('manual')
 581     })
 582
 583     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
 584       this.timeout(80000)
 585
 586       const body = await servers[0].redundancy.listVideos({ target: 'remote-videos' })
 587
 588       const videos = body.data
 589       expect(videos).to.have.lengthOf(1)
 590
 591       const video = videos[0]
 592
 593       for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
 594         await servers[0].redundancy.removeVideo({ redundancyId: r.id })
 595       }
 596
 597       await waitJobs(servers)
 598       await wait(5000)
 599
 600       await check1WebSeed()
 601       await check0PlaylistRedundancies()
 602
 603       await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
 604     })
 605
 606     after(async function () {
 607       await cleanupTests(servers)
 608     })
 609   })
 610
 611   describe('Test expiration', function () {
 612     const strategy = 'recently-added'
 613
 614     async function checkContains (servers: PeerTubeServer[], str: string) {
 615       for (const server of servers) {
 616         const video = await server.videos.get({ id: video1Server2.uuid })
 617
 618         for (const f of video.files) {
 619           expect(f.magnetUri).to.contain(str)
 620         }
 621       }
 622     }
 623
 624     async function checkNotContains (servers: PeerTubeServer[], str: string) {
 625       for (const server of servers) {
 626         const video = await server.videos.get({ id: video1Server2.uuid })
 627
 628         for (const f of video.files) {
 629           expect(f.magnetUri).to.not.contain(str)
 630         }
 631       }
 632     }
 633
 634     before(async function () {
 635       this.timeout(240000)
 636
 637       await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 638
 639       await enableRedundancyOnServer1()
 640     })
 641
 642     it('Should still have 2 webseeds after 10 seconds', async function () {
 643       this.timeout(80000)
 644
 645       await wait(10000)
 646
 647       try {
 648         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 649       } catch {
 650         // Maybe a server deleted a redundancy in the scheduler
 651         await wait(2000)
 652
 653         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 654       }
 655     })
 656
 657     it('Should stop server 1 and expire video redundancy', async function () {
 658       this.timeout(80000)
 659
 660       await killallServers([ servers[0] ])
 661
 662       await wait(15000)
 663
 664       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 665     })
 666
 667     after(async function () {
 668       await cleanupTests(servers)
 669     })
 670   })
 671
 672   describe('Test file replacement', function () {
 673     let video2Server2UUID: string
 674     const strategy = 'recently-added'
 675
 676     before(async function () {
 677       this.timeout(240000)
 678
 679       await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 680
 681       await enableRedundancyOnServer1()
 682
 683       await waitJobs(servers)
 684       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 685       await waitJobs(servers)
 686
 687       await check2Webseeds()
 688       await check1PlaylistRedundancies()
 689       await checkStatsWith1Redundancy(strategy)
 690
 691       const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } })
 692       video2Server2UUID = uuid
 693
 694       // Wait transcoding before federation
 695       await waitJobs(servers)
 696
 697       await servers[1].videos.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
 698     })
 699
 700     it('Should cache video 2 webseeds on the first video', async function () {
 701       this.timeout(240000)
 702
 703       await waitJobs(servers)
 704
 705       let checked = false
 706
 707       while (checked === false) {
 708         await wait(1000)
 709
 710         try {
 711           await check1WebSeed()
 712           await check0PlaylistRedundancies()
 713
 714           await check2Webseeds(video2Server2UUID)
 715           await check1PlaylistRedundancies(video2Server2UUID)
 716
 717           checked = true
 718         } catch {
 719           checked = false
 720         }
 721       }
 722     })
 723
 724     it('Should disable strategy and remove redundancies', async function () {
 725       this.timeout(80000)
 726
 727       await waitJobs(servers)
 728
 729       await killallServers([ servers[0] ])
 730       await servers[0].run({
 731         redundancy: {
 732           videos: {
 733             check_interval: '1 second',
 734             strategies: []
 735           }
 736         }
 737       })
 738
 739       await waitJobs(servers)
 740
 741       await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
 742     })
 743
 744     after(async function () {
 745       await cleanupTests(servers)
 746     })
 747   })
 748 })