server/tests/api/redundancy/redundancy.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { readdir } from 'fs-extra'
   6 import * as magnetUtil from 'magnet-uri'
   7 import { basename, join } from 'path'
   8 import {
   9   checkSegmentHash,
  10   checkVideoFilesWereRemoved,
  11   cleanupTests,
  12   createMultipleServers,
  13   doubleFollow,
  14   killallServers,
  15   makeRawRequest,
  16   PeerTubeServer,
  17   root,
  18   saveVideoInServers,
  19   setAccessTokensToServers,
  20   wait,
  21   waitJobs
  22 } from '@shared/extra-utils'
  23 import {
  24   HttpStatusCode,
  25   VideoDetails,
  26   VideoFile,
  27   VideoPrivacy,
  28   VideoRedundancyStrategy,
  29   VideoRedundancyStrategyWithManual
  30 } from '@shared/models'
  31
  32 const expect = chai.expect
  33
  34 let servers: PeerTubeServer[] = []
  35 let video1Server2: VideoDetails
  36
  37 async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) {
  38   const parsed = magnetUtil.decode(file.magnetUri)
  39
  40   for (const ws of baseWebseeds) {
  41     const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`)
  42     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  43   }
  44
  45   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  46
  47   for (const url of parsed.urlList) {
  48     await makeRawRequest(url, HttpStatusCode.OK_200)
  49   }
  50 }
  51
  52 async function createSingleServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
  53   const strategies: any[] = []
  54
  55   if (strategy !== null) {
  56     strategies.push(
  57       {
  58         min_lifetime: '1 hour',
  59         strategy: strategy,
  60         size: '400KB',
  61
  62         ...additionalParams
  63       }
  64     )
  65   }
  66
  67   const config = {
  68     transcoding: {
  69       webtorrent: {
  70         enabled: withWebtorrent
  71       },
  72       hls: {
  73         enabled: true
  74       }
  75     },
  76     redundancy: {
  77       videos: {
  78         check_interval: '5 seconds',
  79         strategies
  80       }
  81     }
  82   }
  83
  84   servers = await createMultipleServers(3, config)
  85
  86   // Get the access tokens
  87   await setAccessTokensToServers(servers)
  88
  89   {
  90     const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } })
  91     video1Server2 = await servers[1].videos.get({ id })
  92
  93     await servers[1].videos.view({ id })
  94   }
  95
  96   await waitJobs(servers)
  97
  98   // Server 1 and server 2 follow each other
  99   await doubleFollow(servers[0], servers[1])
 100   // Server 1 and server 3 follow each other
 101   await doubleFollow(servers[0], servers[2])
 102   // Server 2 and server 3 follow each other
 103   await doubleFollow(servers[1], servers[2])
 104
 105   await waitJobs(servers)
 106 }
 107
 108 async function ensureSameFilenames (videoUUID: string) {
 109   let webtorrentFilenames: string[]
 110   let hlsFilenames: string[]
 111
 112   for (const server of servers) {
 113     const video = await server.videos.getWithToken({ id: videoUUID })
 114
 115     // Ensure we use the same filenames that the origin
 116
 117     const localWebtorrentFilenames = video.files.map(f => basename(f.fileUrl)).sort()
 118     const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort()
 119
 120     if (webtorrentFilenames) expect(webtorrentFilenames).to.deep.equal(localWebtorrentFilenames)
 121     else webtorrentFilenames = localWebtorrentFilenames
 122
 123     if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames)
 124     else hlsFilenames = localHLSFilenames
 125   }
 126
 127   return { webtorrentFilenames, hlsFilenames }
 128 }
 129
 130 async function check1WebSeed (videoUUID?: string) {
 131   if (!videoUUID) videoUUID = video1Server2.uuid
 132
 133   const webseeds = [
 134     `http://localhost:${servers[1].port}/static/webseed/`
 135   ]
 136
 137   for (const server of servers) {
 138     // With token to avoid issues with video follow constraints
 139     const video = await server.videos.getWithToken({ id: videoUUID })
 140
 141     for (const f of video.files) {
 142       await checkMagnetWebseeds(f, webseeds, server)
 143     }
 144   }
 145
 146   await ensureSameFilenames(videoUUID)
 147 }
 148
 149 async function check2Webseeds (videoUUID?: string) {
 150   if (!videoUUID) videoUUID = video1Server2.uuid
 151
 152   const webseeds = [
 153     `http://localhost:${servers[0].port}/static/redundancy/`,
 154     `http://localhost:${servers[1].port}/static/webseed/`
 155   ]
 156
 157   for (const server of servers) {
 158     const video = await server.videos.get({ id: videoUUID })
 159
 160     for (const file of video.files) {
 161       await checkMagnetWebseeds(file, webseeds, server)
 162     }
 163   }
 164
 165   const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
 166
 167   const directories = [
 168     'test' + servers[0].internalServerNumber + '/redundancy',
 169     'test' + servers[1].internalServerNumber + '/videos'
 170   ]
 171
 172   for (const directory of directories) {
 173     const files = await readdir(join(root(), directory))
 174     expect(files).to.have.length.at.least(4)
 175
 176     // Ensure we files exist on disk
 177     expect(files.find(f => webtorrentFilenames.includes(f))).to.exist
 178   }
 179 }
 180
 181 async function check0PlaylistRedundancies (videoUUID?: string) {
 182   if (!videoUUID) videoUUID = video1Server2.uuid
 183
 184   for (const server of servers) {
 185     // With token to avoid issues with video follow constraints
 186     const video = await server.videos.getWithToken({ id: videoUUID })
 187
 188     expect(video.streamingPlaylists).to.be.an('array')
 189     expect(video.streamingPlaylists).to.have.lengthOf(1)
 190     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 191   }
 192
 193   await ensureSameFilenames(videoUUID)
 194 }
 195
 196 async function check1PlaylistRedundancies (videoUUID?: string) {
 197   if (!videoUUID) videoUUID = video1Server2.uuid
 198
 199   for (const server of servers) {
 200     const video = await server.videos.get({ id: videoUUID })
 201
 202     expect(video.streamingPlaylists).to.have.lengthOf(1)
 203     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 204
 205     const redundancy = video.streamingPlaylists[0].redundancies[0]
 206
 207     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 208   }
 209
 210   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 211   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 212
 213   const video = await servers[0].videos.get({ id: videoUUID })
 214   const hlsPlaylist = video.streamingPlaylists[0]
 215
 216   for (const resolution of [ 240, 360, 480, 720 ]) {
 217     await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist })
 218   }
 219
 220   const { hlsFilenames } = await ensureSameFilenames(videoUUID)
 221
 222   const directories = [
 223     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 224     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 225   ]
 226
 227   for (const directory of directories) {
 228     const files = await readdir(join(root(), directory, videoUUID))
 229     expect(files).to.have.length.at.least(4)
 230
 231     // Ensure we files exist on disk
 232     expect(files.find(f => hlsFilenames.includes(f))).to.exist
 233   }
 234 }
 235
 236 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
 237   let totalSize: number = null
 238   let statsLength = 1
 239
 240   if (strategy !== 'manual') {
 241     totalSize = 409600
 242     statsLength = 2
 243   }
 244
 245   const data = await servers[0].stats.get()
 246   expect(data.videosRedundancy).to.have.lengthOf(statsLength)
 247
 248   const stat = data.videosRedundancy[0]
 249   expect(stat.strategy).to.equal(strategy)
 250   expect(stat.totalSize).to.equal(totalSize)
 251
 252   return stat
 253 }
 254
 255 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
 256   const stat = await checkStatsGlobal(strategy)
 257
 258   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 259   expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
 260   expect(stat.totalVideos).to.equal(1)
 261 }
 262
 263 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
 264   const stat = await checkStatsGlobal(strategy)
 265
 266   expect(stat.totalUsed).to.equal(0)
 267   expect(stat.totalVideoFiles).to.equal(0)
 268   expect(stat.totalVideos).to.equal(0)
 269 }
 270
 271 async function findServerFollows () {
 272   const body = await servers[0].follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
 273   const follows = body.data
 274   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 275   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 276
 277   return { server2, server3 }
 278 }
 279
 280 async function enableRedundancyOnServer1 () {
 281   await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
 282
 283   const { server2, server3 } = await findServerFollows()
 284
 285   expect(server3).to.not.be.undefined
 286   expect(server3.following.hostRedundancyAllowed).to.be.false
 287
 288   expect(server2).to.not.be.undefined
 289   expect(server2.following.hostRedundancyAllowed).to.be.true
 290 }
 291
 292 async function disableRedundancyOnServer1 () {
 293   await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
 294
 295   const { server2, server3 } = await findServerFollows()
 296
 297   expect(server3).to.not.be.undefined
 298   expect(server3.following.hostRedundancyAllowed).to.be.false
 299
 300   expect(server2).to.not.be.undefined
 301   expect(server2.following.hostRedundancyAllowed).to.be.false
 302 }
 303
 304 describe('Test videos redundancy', function () {
 305
 306   describe('With most-views strategy', function () {
 307     const strategy = 'most-views'
 308
 309     before(function () {
 310       this.timeout(120000)
 311
 312       return createSingleServers(strategy)
 313     })
 314
 315     it('Should have 1 webseed on the first video', async function () {
 316       await check1WebSeed()
 317       await check0PlaylistRedundancies()
 318       await checkStatsWithoutRedundancy(strategy)
 319     })
 320
 321     it('Should enable redundancy on server 1', function () {
 322       return enableRedundancyOnServer1()
 323     })
 324
 325     it('Should have 2 webseeds on the first video', async function () {
 326       this.timeout(80000)
 327
 328       await waitJobs(servers)
 329       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 330       await waitJobs(servers)
 331
 332       await check2Webseeds()
 333       await check1PlaylistRedundancies()
 334       await checkStatsWith1Redundancy(strategy)
 335     })
 336
 337     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 338       this.timeout(80000)
 339
 340       await disableRedundancyOnServer1()
 341
 342       await waitJobs(servers)
 343       await wait(5000)
 344
 345       await check1WebSeed()
 346       await check0PlaylistRedundancies()
 347
 348       await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
 349     })
 350
 351     after(async function () {
 352       return cleanupTests(servers)
 353     })
 354   })
 355
 356   describe('With trending strategy', function () {
 357     const strategy = 'trending'
 358
 359     before(function () {
 360       this.timeout(120000)
 361
 362       return createSingleServers(strategy)
 363     })
 364
 365     it('Should have 1 webseed on the first video', async function () {
 366       await check1WebSeed()
 367       await check0PlaylistRedundancies()
 368       await checkStatsWithoutRedundancy(strategy)
 369     })
 370
 371     it('Should enable redundancy on server 1', function () {
 372       return enableRedundancyOnServer1()
 373     })
 374
 375     it('Should have 2 webseeds on the first video', async function () {
 376       this.timeout(80000)
 377
 378       await waitJobs(servers)
 379       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 380       await waitJobs(servers)
 381
 382       await check2Webseeds()
 383       await check1PlaylistRedundancies()
 384       await checkStatsWith1Redundancy(strategy)
 385     })
 386
 387     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 388       this.timeout(80000)
 389
 390       await servers[0].follows.unfollow({ target: servers[1] })
 391
 392       await waitJobs(servers)
 393       await wait(5000)
 394
 395       await check1WebSeed()
 396       await check0PlaylistRedundancies()
 397
 398       await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
 399     })
 400
 401     after(async function () {
 402       await cleanupTests(servers)
 403     })
 404   })
 405
 406   describe('With recently added strategy', function () {
 407     const strategy = 'recently-added'
 408
 409     before(function () {
 410       this.timeout(120000)
 411
 412       return createSingleServers(strategy, { min_views: 3 })
 413     })
 414
 415     it('Should have 1 webseed on the first video', async function () {
 416       await check1WebSeed()
 417       await check0PlaylistRedundancies()
 418       await checkStatsWithoutRedundancy(strategy)
 419     })
 420
 421     it('Should enable redundancy on server 1', function () {
 422       return enableRedundancyOnServer1()
 423     })
 424
 425     it('Should still have 1 webseed on the first video', async function () {
 426       this.timeout(80000)
 427
 428       await waitJobs(servers)
 429       await wait(15000)
 430       await waitJobs(servers)
 431
 432       await check1WebSeed()
 433       await check0PlaylistRedundancies()
 434       await checkStatsWithoutRedundancy(strategy)
 435     })
 436
 437     it('Should view 2 times the first video to have > min_views config', async function () {
 438       this.timeout(80000)
 439
 440       await servers[0].videos.view({ id: video1Server2.uuid })
 441       await servers[2].videos.view({ id: video1Server2.uuid })
 442
 443       await wait(10000)
 444       await waitJobs(servers)
 445     })
 446
 447     it('Should have 2 webseeds on the first video', async function () {
 448       this.timeout(80000)
 449
 450       await waitJobs(servers)
 451       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 452       await waitJobs(servers)
 453
 454       await check2Webseeds()
 455       await check1PlaylistRedundancies()
 456       await checkStatsWith1Redundancy(strategy)
 457     })
 458
 459     it('Should remove the video and the redundancy files', async function () {
 460       this.timeout(20000)
 461
 462       await saveVideoInServers(servers, video1Server2.uuid)
 463       await servers[1].videos.remove({ id: video1Server2.uuid })
 464
 465       await waitJobs(servers)
 466
 467       for (const server of servers) {
 468         await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails })
 469       }
 470     })
 471
 472     after(async function () {
 473       await cleanupTests(servers)
 474     })
 475   })
 476
 477   describe('With only HLS files', function () {
 478     const strategy = 'recently-added'
 479
 480     before(async function () {
 481       this.timeout(120000)
 482
 483       await createSingleServers(strategy, { min_views: 3 }, false)
 484     })
 485
 486     it('Should have 0 playlist redundancy on the first video', async function () {
 487       await check1WebSeed()
 488       await check0PlaylistRedundancies()
 489     })
 490
 491     it('Should enable redundancy on server 1', function () {
 492       return enableRedundancyOnServer1()
 493     })
 494
 495     it('Should still have 0 redundancy on the first video', async function () {
 496       this.timeout(80000)
 497
 498       await waitJobs(servers)
 499       await wait(15000)
 500       await waitJobs(servers)
 501
 502       await check0PlaylistRedundancies()
 503       await checkStatsWithoutRedundancy(strategy)
 504     })
 505
 506     it('Should have 1 redundancy on the first video', async function () {
 507       this.timeout(160000)
 508
 509       await servers[0].videos.view({ id: video1Server2.uuid })
 510       await servers[2].videos.view({ id: video1Server2.uuid })
 511
 512       await wait(10000)
 513       await waitJobs(servers)
 514
 515       await waitJobs(servers)
 516       await servers[0].servers.waitUntilLog('Duplicated ', 1)
 517       await waitJobs(servers)
 518
 519       await check1PlaylistRedundancies()
 520       await checkStatsWith1Redundancy(strategy, true)
 521     })
 522
 523     it('Should remove the video and the redundancy files', async function () {
 524       this.timeout(20000)
 525
 526       await saveVideoInServers(servers, video1Server2.uuid)
 527       await servers[1].videos.remove({ id: video1Server2.uuid })
 528
 529       await waitJobs(servers)
 530
 531       for (const server of servers) {
 532         await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails })
 533       }
 534     })
 535
 536     after(async function () {
 537       await cleanupTests(servers)
 538     })
 539   })
 540
 541   describe('With manual strategy', function () {
 542     before(function () {
 543       this.timeout(120000)
 544
 545       return createSingleServers(null)
 546     })
 547
 548     it('Should have 1 webseed on the first video', async function () {
 549       await check1WebSeed()
 550       await check0PlaylistRedundancies()
 551       await checkStatsWithoutRedundancy('manual')
 552     })
 553
 554     it('Should create a redundancy on first video', async function () {
 555       await servers[0].redundancy.addVideo({ videoId: video1Server2.id })
 556     })
 557
 558     it('Should have 2 webseeds on the first video', async function () {
 559       this.timeout(80000)
 560
 561       await waitJobs(servers)
 562       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 563       await waitJobs(servers)
 564
 565       await check2Webseeds()
 566       await check1PlaylistRedundancies()
 567       await checkStatsWith1Redundancy('manual')
 568     })
 569
 570     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
 571       this.timeout(80000)
 572
 573       const body = await servers[0].redundancy.listVideos({ target: 'remote-videos' })
 574
 575       const videos = body.data
 576       expect(videos).to.have.lengthOf(1)
 577
 578       const video = videos[0]
 579
 580       for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
 581         await servers[0].redundancy.removeVideo({ redundancyId: r.id })
 582       }
 583
 584       await waitJobs(servers)
 585       await wait(5000)
 586
 587       await check1WebSeed()
 588       await check0PlaylistRedundancies()
 589
 590       await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
 591     })
 592
 593     after(async function () {
 594       await cleanupTests(servers)
 595     })
 596   })
 597
 598   describe('Test expiration', function () {
 599     const strategy = 'recently-added'
 600
 601     async function checkContains (servers: PeerTubeServer[], str: string) {
 602       for (const server of servers) {
 603         const video = await server.videos.get({ id: video1Server2.uuid })
 604
 605         for (const f of video.files) {
 606           expect(f.magnetUri).to.contain(str)
 607         }
 608       }
 609     }
 610
 611     async function checkNotContains (servers: PeerTubeServer[], str: string) {
 612       for (const server of servers) {
 613         const video = await server.videos.get({ id: video1Server2.uuid })
 614
 615         for (const f of video.files) {
 616           expect(f.magnetUri).to.not.contain(str)
 617         }
 618       }
 619     }
 620
 621     before(async function () {
 622       this.timeout(120000)
 623
 624       await createSingleServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 625
 626       await enableRedundancyOnServer1()
 627     })
 628
 629     it('Should still have 2 webseeds after 10 seconds', async function () {
 630       this.timeout(80000)
 631
 632       await wait(10000)
 633
 634       try {
 635         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 636       } catch {
 637         // Maybe a server deleted a redundancy in the scheduler
 638         await wait(2000)
 639
 640         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 641       }
 642     })
 643
 644     it('Should stop server 1 and expire video redundancy', async function () {
 645       this.timeout(80000)
 646
 647       await killallServers([ servers[0] ])
 648
 649       await wait(15000)
 650
 651       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 652     })
 653
 654     after(async function () {
 655       await cleanupTests(servers)
 656     })
 657   })
 658
 659   describe('Test file replacement', function () {
 660     let video2Server2UUID: string
 661     const strategy = 'recently-added'
 662
 663     before(async function () {
 664       this.timeout(120000)
 665
 666       await createSingleServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 667
 668       await enableRedundancyOnServer1()
 669
 670       await waitJobs(servers)
 671       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 672       await waitJobs(servers)
 673
 674       await check2Webseeds()
 675       await check1PlaylistRedundancies()
 676       await checkStatsWith1Redundancy(strategy)
 677
 678       const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } })
 679       video2Server2UUID = uuid
 680
 681       // Wait transcoding before federation
 682       await waitJobs(servers)
 683
 684       await servers[1].videos.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
 685     })
 686
 687     it('Should cache video 2 webseeds on the first video', async function () {
 688       this.timeout(120000)
 689
 690       await waitJobs(servers)
 691
 692       let checked = false
 693
 694       while (checked === false) {
 695         await wait(1000)
 696
 697         try {
 698           await check1WebSeed()
 699           await check0PlaylistRedundancies()
 700
 701           await check2Webseeds(video2Server2UUID)
 702           await check1PlaylistRedundancies(video2Server2UUID)
 703
 704           checked = true
 705         } catch {
 706           checked = false
 707         }
 708       }
 709     })
 710
 711     it('Should disable strategy and remove redundancies', async function () {
 712       this.timeout(80000)
 713
 714       await waitJobs(servers)
 715
 716       await killallServers([ servers[0] ])
 717       await servers[0].run({
 718         redundancy: {
 719           videos: {
 720             check_interval: '1 second',
 721             strategies: []
 722           }
 723         }
 724       })
 725
 726       await waitJobs(servers)
 727
 728       await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
 729     })
 730
 731     after(async function () {
 732       await cleanupTests(servers)
 733     })
 734   })
 735 })