server/tests/api/redundancy/redundancy.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { readdir } from 'fs-extra'
   6 import * as magnetUtil from 'magnet-uri'
   7 import { join } from 'path'
   8 import { HttpStatusCode } from '@shared/core-utils'
   9 import {
  10   checkSegmentHash,
  11   checkVideoFilesWereRemoved,
  12   cleanupTests,
  13   doubleFollow,
  14   flushAndRunMultipleServers,
  15   killallServers,
  16   makeGetRequest,
  17   reRunServer,
  18   root,
  19   ServerInfo,
  20   setAccessTokensToServers,
  21   wait,
  22   waitJobs
  23 } from '@shared/extra-utils'
  24 import { VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '@shared/models'
  25
  26 const expect = chai.expect
  27
  28 let servers: ServerInfo[] = []
  29 let video1Server2UUID: string
  30 let video1Server2Id: number
  31
  32 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  33   const parsed = magnetUtil.decode(file.magnetUri)
  34
  35   for (const ws of baseWebseeds) {
  36     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  37     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  38   }
  39
  40   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  41 }
  42
  43 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
  44   const strategies: any[] = []
  45
  46   if (strategy !== null) {
  47     strategies.push(
  48       {
  49         min_lifetime: '1 hour',
  50         strategy: strategy,
  51         size: '400KB',
  52
  53         ...additionalParams
  54       }
  55     )
  56   }
  57
  58   const config = {
  59     transcoding: {
  60       webtorrent: {
  61         enabled: withWebtorrent
  62       },
  63       hls: {
  64         enabled: true
  65       }
  66     },
  67     redundancy: {
  68       videos: {
  69         check_interval: '5 seconds',
  70         strategies
  71       }
  72     }
  73   }
  74
  75   servers = await flushAndRunMultipleServers(3, config)
  76
  77   // Get the access tokens
  78   await setAccessTokensToServers(servers)
  79
  80   {
  81     const { uuid, id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } })
  82     video1Server2UUID = uuid
  83     video1Server2Id = id
  84
  85     await servers[1].videos.view({ id: video1Server2UUID })
  86   }
  87
  88   await waitJobs(servers)
  89
  90   // Server 1 and server 2 follow each other
  91   await doubleFollow(servers[0], servers[1])
  92   // Server 1 and server 3 follow each other
  93   await doubleFollow(servers[0], servers[2])
  94   // Server 2 and server 3 follow each other
  95   await doubleFollow(servers[1], servers[2])
  96
  97   await waitJobs(servers)
  98 }
  99
 100 async function check1WebSeed (videoUUID?: string) {
 101   if (!videoUUID) videoUUID = video1Server2UUID
 102
 103   const webseeds = [
 104     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 105   ]
 106
 107   for (const server of servers) {
 108     // With token to avoid issues with video follow constraints
 109     const video = await server.videos.getWithToken({ id: videoUUID })
 110
 111     for (const f of video.files) {
 112       checkMagnetWebseeds(f, webseeds, server)
 113     }
 114   }
 115 }
 116
 117 async function check2Webseeds (videoUUID?: string) {
 118   if (!videoUUID) videoUUID = video1Server2UUID
 119
 120   const webseeds = [
 121     `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
 122     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 123   ]
 124
 125   for (const server of servers) {
 126     const video = await server.videos.get({ id: videoUUID })
 127
 128     for (const file of video.files) {
 129       checkMagnetWebseeds(file, webseeds, server)
 130
 131       await makeGetRequest({
 132         url: servers[0].url,
 133         statusCodeExpected: HttpStatusCode.OK_200,
 134         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 135         contentType: null
 136       })
 137       await makeGetRequest({
 138         url: servers[1].url,
 139         statusCodeExpected: HttpStatusCode.OK_200,
 140         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 141         contentType: null
 142       })
 143     }
 144   }
 145
 146   const directories = [
 147     'test' + servers[0].internalServerNumber + '/redundancy',
 148     'test' + servers[1].internalServerNumber + '/videos'
 149   ]
 150
 151   for (const directory of directories) {
 152     const files = await readdir(join(root(), directory))
 153     expect(files).to.have.length.at.least(4)
 154
 155     for (const resolution of [ 240, 360, 480, 720 ]) {
 156       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 157     }
 158   }
 159 }
 160
 161 async function check0PlaylistRedundancies (videoUUID?: string) {
 162   if (!videoUUID) videoUUID = video1Server2UUID
 163
 164   for (const server of servers) {
 165     // With token to avoid issues with video follow constraints
 166     const video = await server.videos.getWithToken({ id: videoUUID })
 167
 168     expect(video.streamingPlaylists).to.be.an('array')
 169     expect(video.streamingPlaylists).to.have.lengthOf(1)
 170     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 171   }
 172 }
 173
 174 async function check1PlaylistRedundancies (videoUUID?: string) {
 175   if (!videoUUID) videoUUID = video1Server2UUID
 176
 177   for (const server of servers) {
 178     const video = await server.videos.get({ id: videoUUID })
 179
 180     expect(video.streamingPlaylists).to.have.lengthOf(1)
 181     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 182
 183     const redundancy = video.streamingPlaylists[0].redundancies[0]
 184
 185     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 186   }
 187
 188   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 189   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 190
 191   const video = await servers[0].videos.get({ id: videoUUID })
 192   const hlsPlaylist = video.streamingPlaylists[0]
 193
 194   for (const resolution of [ 240, 360, 480, 720 ]) {
 195     await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist })
 196   }
 197
 198   const directories = [
 199     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 200     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 201   ]
 202
 203   for (const directory of directories) {
 204     const files = await readdir(join(root(), directory, videoUUID))
 205     expect(files).to.have.length.at.least(4)
 206
 207     for (const resolution of [ 240, 360, 480, 720 ]) {
 208       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 209
 210       expect(files.find(f => f === filename)).to.not.be.undefined
 211     }
 212   }
 213 }
 214
 215 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
 216   let totalSize: number = null
 217   let statsLength = 1
 218
 219   if (strategy !== 'manual') {
 220     totalSize = 409600
 221     statsLength = 2
 222   }
 223
 224   const data = await servers[0].stats.get()
 225   expect(data.videosRedundancy).to.have.lengthOf(statsLength)
 226
 227   const stat = data.videosRedundancy[0]
 228   expect(stat.strategy).to.equal(strategy)
 229   expect(stat.totalSize).to.equal(totalSize)
 230
 231   return stat
 232 }
 233
 234 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
 235   const stat = await checkStatsGlobal(strategy)
 236
 237   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 238   expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
 239   expect(stat.totalVideos).to.equal(1)
 240 }
 241
 242 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
 243   const stat = await checkStatsGlobal(strategy)
 244
 245   expect(stat.totalUsed).to.equal(0)
 246   expect(stat.totalVideoFiles).to.equal(0)
 247   expect(stat.totalVideos).to.equal(0)
 248 }
 249
 250 async function findServerFollows () {
 251   const body = await servers[0].follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
 252   const follows = body.data
 253   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 254   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 255
 256   return { server2, server3 }
 257 }
 258
 259 async function enableRedundancyOnServer1 () {
 260   await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
 261
 262   const { server2, server3 } = await findServerFollows()
 263
 264   expect(server3).to.not.be.undefined
 265   expect(server3.following.hostRedundancyAllowed).to.be.false
 266
 267   expect(server2).to.not.be.undefined
 268   expect(server2.following.hostRedundancyAllowed).to.be.true
 269 }
 270
 271 async function disableRedundancyOnServer1 () {
 272   await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
 273
 274   const { server2, server3 } = await findServerFollows()
 275
 276   expect(server3).to.not.be.undefined
 277   expect(server3.following.hostRedundancyAllowed).to.be.false
 278
 279   expect(server2).to.not.be.undefined
 280   expect(server2.following.hostRedundancyAllowed).to.be.false
 281 }
 282
 283 describe('Test videos redundancy', function () {
 284
 285   describe('With most-views strategy', function () {
 286     const strategy = 'most-views'
 287
 288     before(function () {
 289       this.timeout(120000)
 290
 291       return flushAndRunServers(strategy)
 292     })
 293
 294     it('Should have 1 webseed on the first video', async function () {
 295       await check1WebSeed()
 296       await check0PlaylistRedundancies()
 297       await checkStatsWithoutRedundancy(strategy)
 298     })
 299
 300     it('Should enable redundancy on server 1', function () {
 301       return enableRedundancyOnServer1()
 302     })
 303
 304     it('Should have 2 webseeds on the first video', async function () {
 305       this.timeout(80000)
 306
 307       await waitJobs(servers)
 308       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 309       await waitJobs(servers)
 310
 311       await check2Webseeds()
 312       await check1PlaylistRedundancies()
 313       await checkStatsWith1Redundancy(strategy)
 314     })
 315
 316     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 317       this.timeout(80000)
 318
 319       await disableRedundancyOnServer1()
 320
 321       await waitJobs(servers)
 322       await wait(5000)
 323
 324       await check1WebSeed()
 325       await check0PlaylistRedundancies()
 326
 327       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos', join('playlists', 'hls') ])
 328     })
 329
 330     after(async function () {
 331       return cleanupTests(servers)
 332     })
 333   })
 334
 335   describe('With trending strategy', function () {
 336     const strategy = 'trending'
 337
 338     before(function () {
 339       this.timeout(120000)
 340
 341       return flushAndRunServers(strategy)
 342     })
 343
 344     it('Should have 1 webseed on the first video', async function () {
 345       await check1WebSeed()
 346       await check0PlaylistRedundancies()
 347       await checkStatsWithoutRedundancy(strategy)
 348     })
 349
 350     it('Should enable redundancy on server 1', function () {
 351       return enableRedundancyOnServer1()
 352     })
 353
 354     it('Should have 2 webseeds on the first video', async function () {
 355       this.timeout(80000)
 356
 357       await waitJobs(servers)
 358       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 359       await waitJobs(servers)
 360
 361       await check2Webseeds()
 362       await check1PlaylistRedundancies()
 363       await checkStatsWith1Redundancy(strategy)
 364     })
 365
 366     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 367       this.timeout(80000)
 368
 369       await servers[0].follows.unfollow({ target: servers[1] })
 370
 371       await waitJobs(servers)
 372       await wait(5000)
 373
 374       await check1WebSeed()
 375       await check0PlaylistRedundancies()
 376
 377       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ])
 378     })
 379
 380     after(async function () {
 381       await cleanupTests(servers)
 382     })
 383   })
 384
 385   describe('With recently added strategy', function () {
 386     const strategy = 'recently-added'
 387
 388     before(function () {
 389       this.timeout(120000)
 390
 391       return flushAndRunServers(strategy, { min_views: 3 })
 392     })
 393
 394     it('Should have 1 webseed on the first video', async function () {
 395       await check1WebSeed()
 396       await check0PlaylistRedundancies()
 397       await checkStatsWithoutRedundancy(strategy)
 398     })
 399
 400     it('Should enable redundancy on server 1', function () {
 401       return enableRedundancyOnServer1()
 402     })
 403
 404     it('Should still have 1 webseed on the first video', async function () {
 405       this.timeout(80000)
 406
 407       await waitJobs(servers)
 408       await wait(15000)
 409       await waitJobs(servers)
 410
 411       await check1WebSeed()
 412       await check0PlaylistRedundancies()
 413       await checkStatsWithoutRedundancy(strategy)
 414     })
 415
 416     it('Should view 2 times the first video to have > min_views config', async function () {
 417       this.timeout(80000)
 418
 419       await servers[0].videos.view({ id: video1Server2UUID })
 420       await servers[2].videos.view({ id: video1Server2UUID })
 421
 422       await wait(10000)
 423       await waitJobs(servers)
 424     })
 425
 426     it('Should have 2 webseeds on the first video', async function () {
 427       this.timeout(80000)
 428
 429       await waitJobs(servers)
 430       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 431       await waitJobs(servers)
 432
 433       await check2Webseeds()
 434       await check1PlaylistRedundancies()
 435       await checkStatsWith1Redundancy(strategy)
 436     })
 437
 438     it('Should remove the video and the redundancy files', async function () {
 439       this.timeout(20000)
 440
 441       await servers[1].videos.remove({ id: video1Server2UUID })
 442
 443       await waitJobs(servers)
 444
 445       for (const server of servers) {
 446         await checkVideoFilesWereRemoved(video1Server2UUID, server)
 447       }
 448     })
 449
 450     after(async function () {
 451       await cleanupTests(servers)
 452     })
 453   })
 454
 455   describe('With only HLS files', function () {
 456     const strategy = 'recently-added'
 457
 458     before(async function () {
 459       this.timeout(120000)
 460
 461       await flushAndRunServers(strategy, { min_views: 3 }, false)
 462     })
 463
 464     it('Should have 0 playlist redundancy on the first video', async function () {
 465       await check1WebSeed()
 466       await check0PlaylistRedundancies()
 467     })
 468
 469     it('Should enable redundancy on server 1', function () {
 470       return enableRedundancyOnServer1()
 471     })
 472
 473     it('Should still have 0 redundancy on the first video', async function () {
 474       this.timeout(80000)
 475
 476       await waitJobs(servers)
 477       await wait(15000)
 478       await waitJobs(servers)
 479
 480       await check0PlaylistRedundancies()
 481       await checkStatsWithoutRedundancy(strategy)
 482     })
 483
 484     it('Should have 1 redundancy on the first video', async function () {
 485       this.timeout(160000)
 486
 487       await servers[0].videos.view({ id: video1Server2UUID })
 488       await servers[2].videos.view({ id: video1Server2UUID })
 489
 490       await wait(10000)
 491       await waitJobs(servers)
 492
 493       await waitJobs(servers)
 494       await servers[0].servers.waitUntilLog('Duplicated ', 1)
 495       await waitJobs(servers)
 496
 497       await check1PlaylistRedundancies()
 498       await checkStatsWith1Redundancy(strategy, true)
 499     })
 500
 501     it('Should remove the video and the redundancy files', async function () {
 502       this.timeout(20000)
 503
 504       await servers[1].videos.remove({ id: video1Server2UUID })
 505
 506       await waitJobs(servers)
 507
 508       for (const server of servers) {
 509         await checkVideoFilesWereRemoved(video1Server2UUID, server)
 510       }
 511     })
 512
 513     after(async function () {
 514       await cleanupTests(servers)
 515     })
 516   })
 517
 518   describe('With manual strategy', function () {
 519     before(function () {
 520       this.timeout(120000)
 521
 522       return flushAndRunServers(null)
 523     })
 524
 525     it('Should have 1 webseed on the first video', async function () {
 526       await check1WebSeed()
 527       await check0PlaylistRedundancies()
 528       await checkStatsWithoutRedundancy('manual')
 529     })
 530
 531     it('Should create a redundancy on first video', async function () {
 532       await servers[0].redundancy.addVideo({ videoId: video1Server2Id })
 533     })
 534
 535     it('Should have 2 webseeds on the first video', async function () {
 536       this.timeout(80000)
 537
 538       await waitJobs(servers)
 539       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 540       await waitJobs(servers)
 541
 542       await check2Webseeds()
 543       await check1PlaylistRedundancies()
 544       await checkStatsWith1Redundancy('manual')
 545     })
 546
 547     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
 548       this.timeout(80000)
 549
 550       const body = await servers[0].redundancy.listVideos({ target: 'remote-videos' })
 551
 552       const videos = body.data
 553       expect(videos).to.have.lengthOf(1)
 554
 555       const video = videos[0]
 556
 557       for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
 558         await servers[0].redundancy.removeVideo({ redundancyId: r.id })
 559       }
 560
 561       await waitJobs(servers)
 562       await wait(5000)
 563
 564       await check1WebSeed()
 565       await check0PlaylistRedundancies()
 566
 567       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ])
 568     })
 569
 570     after(async function () {
 571       await cleanupTests(servers)
 572     })
 573   })
 574
 575   describe('Test expiration', function () {
 576     const strategy = 'recently-added'
 577
 578     async function checkContains (servers: ServerInfo[], str: string) {
 579       for (const server of servers) {
 580         const video = await server.videos.get({ id: video1Server2UUID })
 581
 582         for (const f of video.files) {
 583           expect(f.magnetUri).to.contain(str)
 584         }
 585       }
 586     }
 587
 588     async function checkNotContains (servers: ServerInfo[], str: string) {
 589       for (const server of servers) {
 590         const video = await server.videos.get({ id: video1Server2UUID })
 591
 592         for (const f of video.files) {
 593           expect(f.magnetUri).to.not.contain(str)
 594         }
 595       }
 596     }
 597
 598     before(async function () {
 599       this.timeout(120000)
 600
 601       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 602
 603       await enableRedundancyOnServer1()
 604     })
 605
 606     it('Should still have 2 webseeds after 10 seconds', async function () {
 607       this.timeout(80000)
 608
 609       await wait(10000)
 610
 611       try {
 612         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 613       } catch {
 614         // Maybe a server deleted a redundancy in the scheduler
 615         await wait(2000)
 616
 617         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 618       }
 619     })
 620
 621     it('Should stop server 1 and expire video redundancy', async function () {
 622       this.timeout(80000)
 623
 624       await killallServers([ servers[0] ])
 625
 626       await wait(15000)
 627
 628       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 629     })
 630
 631     after(async function () {
 632       await cleanupTests(servers)
 633     })
 634   })
 635
 636   describe('Test file replacement', function () {
 637     let video2Server2UUID: string
 638     const strategy = 'recently-added'
 639
 640     before(async function () {
 641       this.timeout(120000)
 642
 643       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 644
 645       await enableRedundancyOnServer1()
 646
 647       await waitJobs(servers)
 648       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 649       await waitJobs(servers)
 650
 651       await check2Webseeds(video1Server2UUID)
 652       await check1PlaylistRedundancies(video1Server2UUID)
 653       await checkStatsWith1Redundancy(strategy)
 654
 655       const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } })
 656       video2Server2UUID = uuid
 657
 658       // Wait transcoding before federation
 659       await waitJobs(servers)
 660
 661       await servers[1].videos.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
 662     })
 663
 664     it('Should cache video 2 webseeds on the first video', async function () {
 665       this.timeout(120000)
 666
 667       await waitJobs(servers)
 668
 669       let checked = false
 670
 671       while (checked === false) {
 672         await wait(1000)
 673
 674         try {
 675           await check1WebSeed(video1Server2UUID)
 676           await check0PlaylistRedundancies(video1Server2UUID)
 677
 678           await check2Webseeds(video2Server2UUID)
 679           await check1PlaylistRedundancies(video2Server2UUID)
 680
 681           checked = true
 682         } catch {
 683           checked = false
 684         }
 685       }
 686     })
 687
 688     it('Should disable strategy and remove redundancies', async function () {
 689       this.timeout(80000)
 690
 691       await waitJobs(servers)
 692
 693       await killallServers([ servers[0] ])
 694       await reRunServer(servers[0], {
 695         redundancy: {
 696           videos: {
 697             check_interval: '1 second',
 698             strategies: []
 699           }
 700         }
 701       })
 702
 703       await waitJobs(servers)
 704
 705       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ join('redundancy', 'hls') ])
 706     })
 707
 708     after(async function () {
 709       await cleanupTests(servers)
 710     })
 711   })
 712 })