server/tests/api/redundancy/redundancy.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { readdir } from 'fs-extra'
   6 import * as magnetUtil from 'magnet-uri'
   7 import { join } from 'path'
   8 import { HttpStatusCode } from '@shared/core-utils'
   9 import {
  10   checkSegmentHash,
  11   checkVideoFilesWereRemoved,
  12   cleanupTests,
  13   doubleFollow,
  14   createMultipleServers,
  15   killallServers,
  16   makeGetRequest,
  17   root,
  18   PeerTubeServer,
  19   setAccessTokensToServers,
  20   wait,
  21   waitJobs
  22 } from '@shared/extra-utils'
  23 import { VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '@shared/models'
  24
  25 const expect = chai.expect
  26
  27 let servers: PeerTubeServer[] = []
  28 let video1Server2UUID: string
  29 let video1Server2Id: number
  30
  31 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: PeerTubeServer) {
  32   const parsed = magnetUtil.decode(file.magnetUri)
  33
  34   for (const ws of baseWebseeds) {
  35     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  36     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  37   }
  38
  39   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  40 }
  41
  42 async function createSingleServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
  43   const strategies: any[] = []
  44
  45   if (strategy !== null) {
  46     strategies.push(
  47       {
  48         min_lifetime: '1 hour',
  49         strategy: strategy,
  50         size: '400KB',
  51
  52         ...additionalParams
  53       }
  54     )
  55   }
  56
  57   const config = {
  58     transcoding: {
  59       webtorrent: {
  60         enabled: withWebtorrent
  61       },
  62       hls: {
  63         enabled: true
  64       }
  65     },
  66     redundancy: {
  67       videos: {
  68         check_interval: '5 seconds',
  69         strategies
  70       }
  71     }
  72   }
  73
  74   servers = await createMultipleServers(3, config)
  75
  76   // Get the access tokens
  77   await setAccessTokensToServers(servers)
  78
  79   {
  80     const { uuid, id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } })
  81     video1Server2UUID = uuid
  82     video1Server2Id = id
  83
  84     await servers[1].videos.view({ id: video1Server2UUID })
  85   }
  86
  87   await waitJobs(servers)
  88
  89   // Server 1 and server 2 follow each other
  90   await doubleFollow(servers[0], servers[1])
  91   // Server 1 and server 3 follow each other
  92   await doubleFollow(servers[0], servers[2])
  93   // Server 2 and server 3 follow each other
  94   await doubleFollow(servers[1], servers[2])
  95
  96   await waitJobs(servers)
  97 }
  98
  99 async function check1WebSeed (videoUUID?: string) {
 100   if (!videoUUID) videoUUID = video1Server2UUID
 101
 102   const webseeds = [
 103     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 104   ]
 105
 106   for (const server of servers) {
 107     // With token to avoid issues with video follow constraints
 108     const video = await server.videos.getWithToken({ id: videoUUID })
 109
 110     for (const f of video.files) {
 111       checkMagnetWebseeds(f, webseeds, server)
 112     }
 113   }
 114 }
 115
 116 async function check2Webseeds (videoUUID?: string) {
 117   if (!videoUUID) videoUUID = video1Server2UUID
 118
 119   const webseeds = [
 120     `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
 121     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 122   ]
 123
 124   for (const server of servers) {
 125     const video = await server.videos.get({ id: videoUUID })
 126
 127     for (const file of video.files) {
 128       checkMagnetWebseeds(file, webseeds, server)
 129
 130       await makeGetRequest({
 131         url: servers[0].url,
 132         statusCodeExpected: HttpStatusCode.OK_200,
 133         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 134         contentType: null
 135       })
 136       await makeGetRequest({
 137         url: servers[1].url,
 138         statusCodeExpected: HttpStatusCode.OK_200,
 139         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 140         contentType: null
 141       })
 142     }
 143   }
 144
 145   const directories = [
 146     'test' + servers[0].internalServerNumber + '/redundancy',
 147     'test' + servers[1].internalServerNumber + '/videos'
 148   ]
 149
 150   for (const directory of directories) {
 151     const files = await readdir(join(root(), directory))
 152     expect(files).to.have.length.at.least(4)
 153
 154     for (const resolution of [ 240, 360, 480, 720 ]) {
 155       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 156     }
 157   }
 158 }
 159
 160 async function check0PlaylistRedundancies (videoUUID?: string) {
 161   if (!videoUUID) videoUUID = video1Server2UUID
 162
 163   for (const server of servers) {
 164     // With token to avoid issues with video follow constraints
 165     const video = await server.videos.getWithToken({ id: videoUUID })
 166
 167     expect(video.streamingPlaylists).to.be.an('array')
 168     expect(video.streamingPlaylists).to.have.lengthOf(1)
 169     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 170   }
 171 }
 172
 173 async function check1PlaylistRedundancies (videoUUID?: string) {
 174   if (!videoUUID) videoUUID = video1Server2UUID
 175
 176   for (const server of servers) {
 177     const video = await server.videos.get({ id: videoUUID })
 178
 179     expect(video.streamingPlaylists).to.have.lengthOf(1)
 180     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 181
 182     const redundancy = video.streamingPlaylists[0].redundancies[0]
 183
 184     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 185   }
 186
 187   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 188   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 189
 190   const video = await servers[0].videos.get({ id: videoUUID })
 191   const hlsPlaylist = video.streamingPlaylists[0]
 192
 193   for (const resolution of [ 240, 360, 480, 720 ]) {
 194     await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist })
 195   }
 196
 197   const directories = [
 198     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 199     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 200   ]
 201
 202   for (const directory of directories) {
 203     const files = await readdir(join(root(), directory, videoUUID))
 204     expect(files).to.have.length.at.least(4)
 205
 206     for (const resolution of [ 240, 360, 480, 720 ]) {
 207       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 208
 209       expect(files.find(f => f === filename)).to.not.be.undefined
 210     }
 211   }
 212 }
 213
 214 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
 215   let totalSize: number = null
 216   let statsLength = 1
 217
 218   if (strategy !== 'manual') {
 219     totalSize = 409600
 220     statsLength = 2
 221   }
 222
 223   const data = await servers[0].stats.get()
 224   expect(data.videosRedundancy).to.have.lengthOf(statsLength)
 225
 226   const stat = data.videosRedundancy[0]
 227   expect(stat.strategy).to.equal(strategy)
 228   expect(stat.totalSize).to.equal(totalSize)
 229
 230   return stat
 231 }
 232
 233 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
 234   const stat = await checkStatsGlobal(strategy)
 235
 236   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 237   expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
 238   expect(stat.totalVideos).to.equal(1)
 239 }
 240
 241 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
 242   const stat = await checkStatsGlobal(strategy)
 243
 244   expect(stat.totalUsed).to.equal(0)
 245   expect(stat.totalVideoFiles).to.equal(0)
 246   expect(stat.totalVideos).to.equal(0)
 247 }
 248
 249 async function findServerFollows () {
 250   const body = await servers[0].follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
 251   const follows = body.data
 252   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 253   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 254
 255   return { server2, server3 }
 256 }
 257
 258 async function enableRedundancyOnServer1 () {
 259   await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
 260
 261   const { server2, server3 } = await findServerFollows()
 262
 263   expect(server3).to.not.be.undefined
 264   expect(server3.following.hostRedundancyAllowed).to.be.false
 265
 266   expect(server2).to.not.be.undefined
 267   expect(server2.following.hostRedundancyAllowed).to.be.true
 268 }
 269
 270 async function disableRedundancyOnServer1 () {
 271   await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
 272
 273   const { server2, server3 } = await findServerFollows()
 274
 275   expect(server3).to.not.be.undefined
 276   expect(server3.following.hostRedundancyAllowed).to.be.false
 277
 278   expect(server2).to.not.be.undefined
 279   expect(server2.following.hostRedundancyAllowed).to.be.false
 280 }
 281
 282 describe('Test videos redundancy', function () {
 283
 284   describe('With most-views strategy', function () {
 285     const strategy = 'most-views'
 286
 287     before(function () {
 288       this.timeout(120000)
 289
 290       return createSingleServers(strategy)
 291     })
 292
 293     it('Should have 1 webseed on the first video', async function () {
 294       await check1WebSeed()
 295       await check0PlaylistRedundancies()
 296       await checkStatsWithoutRedundancy(strategy)
 297     })
 298
 299     it('Should enable redundancy on server 1', function () {
 300       return enableRedundancyOnServer1()
 301     })
 302
 303     it('Should have 2 webseeds on the first video', async function () {
 304       this.timeout(80000)
 305
 306       await waitJobs(servers)
 307       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 308       await waitJobs(servers)
 309
 310       await check2Webseeds()
 311       await check1PlaylistRedundancies()
 312       await checkStatsWith1Redundancy(strategy)
 313     })
 314
 315     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 316       this.timeout(80000)
 317
 318       await disableRedundancyOnServer1()
 319
 320       await waitJobs(servers)
 321       await wait(5000)
 322
 323       await check1WebSeed()
 324       await check0PlaylistRedundancies()
 325
 326       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos', join('playlists', 'hls') ])
 327     })
 328
 329     after(async function () {
 330       return cleanupTests(servers)
 331     })
 332   })
 333
 334   describe('With trending strategy', function () {
 335     const strategy = 'trending'
 336
 337     before(function () {
 338       this.timeout(120000)
 339
 340       return createSingleServers(strategy)
 341     })
 342
 343     it('Should have 1 webseed on the first video', async function () {
 344       await check1WebSeed()
 345       await check0PlaylistRedundancies()
 346       await checkStatsWithoutRedundancy(strategy)
 347     })
 348
 349     it('Should enable redundancy on server 1', function () {
 350       return enableRedundancyOnServer1()
 351     })
 352
 353     it('Should have 2 webseeds on the first video', async function () {
 354       this.timeout(80000)
 355
 356       await waitJobs(servers)
 357       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 358       await waitJobs(servers)
 359
 360       await check2Webseeds()
 361       await check1PlaylistRedundancies()
 362       await checkStatsWith1Redundancy(strategy)
 363     })
 364
 365     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 366       this.timeout(80000)
 367
 368       await servers[0].follows.unfollow({ target: servers[1] })
 369
 370       await waitJobs(servers)
 371       await wait(5000)
 372
 373       await check1WebSeed()
 374       await check0PlaylistRedundancies()
 375
 376       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ])
 377     })
 378
 379     after(async function () {
 380       await cleanupTests(servers)
 381     })
 382   })
 383
 384   describe('With recently added strategy', function () {
 385     const strategy = 'recently-added'
 386
 387     before(function () {
 388       this.timeout(120000)
 389
 390       return createSingleServers(strategy, { min_views: 3 })
 391     })
 392
 393     it('Should have 1 webseed on the first video', async function () {
 394       await check1WebSeed()
 395       await check0PlaylistRedundancies()
 396       await checkStatsWithoutRedundancy(strategy)
 397     })
 398
 399     it('Should enable redundancy on server 1', function () {
 400       return enableRedundancyOnServer1()
 401     })
 402
 403     it('Should still have 1 webseed on the first video', async function () {
 404       this.timeout(80000)
 405
 406       await waitJobs(servers)
 407       await wait(15000)
 408       await waitJobs(servers)
 409
 410       await check1WebSeed()
 411       await check0PlaylistRedundancies()
 412       await checkStatsWithoutRedundancy(strategy)
 413     })
 414
 415     it('Should view 2 times the first video to have > min_views config', async function () {
 416       this.timeout(80000)
 417
 418       await servers[0].videos.view({ id: video1Server2UUID })
 419       await servers[2].videos.view({ id: video1Server2UUID })
 420
 421       await wait(10000)
 422       await waitJobs(servers)
 423     })
 424
 425     it('Should have 2 webseeds on the first video', async function () {
 426       this.timeout(80000)
 427
 428       await waitJobs(servers)
 429       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 430       await waitJobs(servers)
 431
 432       await check2Webseeds()
 433       await check1PlaylistRedundancies()
 434       await checkStatsWith1Redundancy(strategy)
 435     })
 436
 437     it('Should remove the video and the redundancy files', async function () {
 438       this.timeout(20000)
 439
 440       await servers[1].videos.remove({ id: video1Server2UUID })
 441
 442       await waitJobs(servers)
 443
 444       for (const server of servers) {
 445         await checkVideoFilesWereRemoved(video1Server2UUID, server)
 446       }
 447     })
 448
 449     after(async function () {
 450       await cleanupTests(servers)
 451     })
 452   })
 453
 454   describe('With only HLS files', function () {
 455     const strategy = 'recently-added'
 456
 457     before(async function () {
 458       this.timeout(120000)
 459
 460       await createSingleServers(strategy, { min_views: 3 }, false)
 461     })
 462
 463     it('Should have 0 playlist redundancy on the first video', async function () {
 464       await check1WebSeed()
 465       await check0PlaylistRedundancies()
 466     })
 467
 468     it('Should enable redundancy on server 1', function () {
 469       return enableRedundancyOnServer1()
 470     })
 471
 472     it('Should still have 0 redundancy on the first video', async function () {
 473       this.timeout(80000)
 474
 475       await waitJobs(servers)
 476       await wait(15000)
 477       await waitJobs(servers)
 478
 479       await check0PlaylistRedundancies()
 480       await checkStatsWithoutRedundancy(strategy)
 481     })
 482
 483     it('Should have 1 redundancy on the first video', async function () {
 484       this.timeout(160000)
 485
 486       await servers[0].videos.view({ id: video1Server2UUID })
 487       await servers[2].videos.view({ id: video1Server2UUID })
 488
 489       await wait(10000)
 490       await waitJobs(servers)
 491
 492       await waitJobs(servers)
 493       await servers[0].servers.waitUntilLog('Duplicated ', 1)
 494       await waitJobs(servers)
 495
 496       await check1PlaylistRedundancies()
 497       await checkStatsWith1Redundancy(strategy, true)
 498     })
 499
 500     it('Should remove the video and the redundancy files', async function () {
 501       this.timeout(20000)
 502
 503       await servers[1].videos.remove({ id: video1Server2UUID })
 504
 505       await waitJobs(servers)
 506
 507       for (const server of servers) {
 508         await checkVideoFilesWereRemoved(video1Server2UUID, server)
 509       }
 510     })
 511
 512     after(async function () {
 513       await cleanupTests(servers)
 514     })
 515   })
 516
 517   describe('With manual strategy', function () {
 518     before(function () {
 519       this.timeout(120000)
 520
 521       return createSingleServers(null)
 522     })
 523
 524     it('Should have 1 webseed on the first video', async function () {
 525       await check1WebSeed()
 526       await check0PlaylistRedundancies()
 527       await checkStatsWithoutRedundancy('manual')
 528     })
 529
 530     it('Should create a redundancy on first video', async function () {
 531       await servers[0].redundancy.addVideo({ videoId: video1Server2Id })
 532     })
 533
 534     it('Should have 2 webseeds on the first video', async function () {
 535       this.timeout(80000)
 536
 537       await waitJobs(servers)
 538       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 539       await waitJobs(servers)
 540
 541       await check2Webseeds()
 542       await check1PlaylistRedundancies()
 543       await checkStatsWith1Redundancy('manual')
 544     })
 545
 546     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
 547       this.timeout(80000)
 548
 549       const body = await servers[0].redundancy.listVideos({ target: 'remote-videos' })
 550
 551       const videos = body.data
 552       expect(videos).to.have.lengthOf(1)
 553
 554       const video = videos[0]
 555
 556       for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
 557         await servers[0].redundancy.removeVideo({ redundancyId: r.id })
 558       }
 559
 560       await waitJobs(servers)
 561       await wait(5000)
 562
 563       await check1WebSeed()
 564       await check0PlaylistRedundancies()
 565
 566       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ])
 567     })
 568
 569     after(async function () {
 570       await cleanupTests(servers)
 571     })
 572   })
 573
 574   describe('Test expiration', function () {
 575     const strategy = 'recently-added'
 576
 577     async function checkContains (servers: PeerTubeServer[], str: string) {
 578       for (const server of servers) {
 579         const video = await server.videos.get({ id: video1Server2UUID })
 580
 581         for (const f of video.files) {
 582           expect(f.magnetUri).to.contain(str)
 583         }
 584       }
 585     }
 586
 587     async function checkNotContains (servers: PeerTubeServer[], str: string) {
 588       for (const server of servers) {
 589         const video = await server.videos.get({ id: video1Server2UUID })
 590
 591         for (const f of video.files) {
 592           expect(f.magnetUri).to.not.contain(str)
 593         }
 594       }
 595     }
 596
 597     before(async function () {
 598       this.timeout(120000)
 599
 600       await createSingleServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 601
 602       await enableRedundancyOnServer1()
 603     })
 604
 605     it('Should still have 2 webseeds after 10 seconds', async function () {
 606       this.timeout(80000)
 607
 608       await wait(10000)
 609
 610       try {
 611         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 612       } catch {
 613         // Maybe a server deleted a redundancy in the scheduler
 614         await wait(2000)
 615
 616         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 617       }
 618     })
 619
 620     it('Should stop server 1 and expire video redundancy', async function () {
 621       this.timeout(80000)
 622
 623       await killallServers([ servers[0] ])
 624
 625       await wait(15000)
 626
 627       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 628     })
 629
 630     after(async function () {
 631       await cleanupTests(servers)
 632     })
 633   })
 634
 635   describe('Test file replacement', function () {
 636     let video2Server2UUID: string
 637     const strategy = 'recently-added'
 638
 639     before(async function () {
 640       this.timeout(120000)
 641
 642       await createSingleServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 643
 644       await enableRedundancyOnServer1()
 645
 646       await waitJobs(servers)
 647       await servers[0].servers.waitUntilLog('Duplicated ', 5)
 648       await waitJobs(servers)
 649
 650       await check2Webseeds(video1Server2UUID)
 651       await check1PlaylistRedundancies(video1Server2UUID)
 652       await checkStatsWith1Redundancy(strategy)
 653
 654       const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } })
 655       video2Server2UUID = uuid
 656
 657       // Wait transcoding before federation
 658       await waitJobs(servers)
 659
 660       await servers[1].videos.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
 661     })
 662
 663     it('Should cache video 2 webseeds on the first video', async function () {
 664       this.timeout(120000)
 665
 666       await waitJobs(servers)
 667
 668       let checked = false
 669
 670       while (checked === false) {
 671         await wait(1000)
 672
 673         try {
 674           await check1WebSeed(video1Server2UUID)
 675           await check0PlaylistRedundancies(video1Server2UUID)
 676
 677           await check2Webseeds(video2Server2UUID)
 678           await check1PlaylistRedundancies(video2Server2UUID)
 679
 680           checked = true
 681         } catch {
 682           checked = false
 683         }
 684       }
 685     })
 686
 687     it('Should disable strategy and remove redundancies', async function () {
 688       this.timeout(80000)
 689
 690       await waitJobs(servers)
 691
 692       await killallServers([ servers[0] ])
 693       await servers[0].run({
 694         redundancy: {
 695           videos: {
 696             check_interval: '1 second',
 697             strategies: []
 698           }
 699         }
 700       })
 701
 702       await waitJobs(servers)
 703
 704       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ join('redundancy', 'hls') ])
 705     })
 706
 707     after(async function () {
 708       await cleanupTests(servers)
 709     })
 710   })
 711 })