server/tests/api/redundancy/redundancy.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   checkSegmentHash,
   8   checkVideoFilesWereRemoved,
   9   cleanupTests,
  10   doubleFollow,
  11   flushAndRunMultipleServers,
  12   getFollowingListPaginationAndSort,
  13   getVideo,
  14   getVideoWithToken,
  15   immutableAssign,
  16   killallServers,
  17   makeGetRequest,
  18   removeVideo,
  19   reRunServer,
  20   root,
  21   ServerInfo,
  22   setAccessTokensToServers,
  23   unfollow,
  24   uploadVideo,
  25   viewVideo,
  26   wait,
  27   waitUntilLog
  28 } from '../../../../shared/extra-utils'
  29 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
  30
  31 import * as magnetUtil from 'magnet-uri'
  32 import {
  33   addVideoRedundancy,
  34   listVideoRedundancies,
  35   removeVideoRedundancy,
  36   updateRedundancy
  37 } from '../../../../shared/extra-utils/server/redundancy'
  38 import { ActorFollow } from '../../../../shared/models/actors'
  39 import { readdir } from 'fs-extra'
  40 import { join } from 'path'
  41 import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
  42 import { getStats } from '../../../../shared/extra-utils/server/stats'
  43 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  44
  45 const expect = chai.expect
  46
  47 let servers: ServerInfo[] = []
  48 let video1Server2UUID: string
  49 let video1Server2Id: number
  50
  51 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  52   const parsed = magnetUtil.decode(file.magnetUri)
  53
  54   for (const ws of baseWebseeds) {
  55     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  56     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  57   }
  58
  59   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  60 }
  61
  62 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) {
  63   const strategies: any[] = []
  64
  65   if (strategy !== null) {
  66     strategies.push(
  67       immutableAssign({
  68         min_lifetime: '1 hour',
  69         strategy: strategy,
  70         size: '400KB'
  71       }, additionalParams)
  72     )
  73   }
  74
  75   const config = {
  76     transcoding: {
  77       hls: {
  78         enabled: true
  79       }
  80     },
  81     redundancy: {
  82       videos: {
  83         check_interval: '5 seconds',
  84         strategies
  85       }
  86     }
  87   }
  88
  89   servers = await flushAndRunMultipleServers(3, config)
  90
  91   // Get the access tokens
  92   await setAccessTokensToServers(servers)
  93
  94   {
  95     const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
  96     video1Server2UUID = res.body.video.uuid
  97     video1Server2Id = res.body.video.id
  98
  99     await viewVideo(servers[1].url, video1Server2UUID)
 100   }
 101
 102   await waitJobs(servers)
 103
 104   // Server 1 and server 2 follow each other
 105   await doubleFollow(servers[0], servers[1])
 106   // Server 1 and server 3 follow each other
 107   await doubleFollow(servers[0], servers[2])
 108   // Server 2 and server 3 follow each other
 109   await doubleFollow(servers[1], servers[2])
 110
 111   await waitJobs(servers)
 112 }
 113
 114 async function check1WebSeed (videoUUID?: string) {
 115   if (!videoUUID) videoUUID = video1Server2UUID
 116
 117   const webseeds = [
 118     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 119   ]
 120
 121   for (const server of servers) {
 122     // With token to avoid issues with video follow constraints
 123     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 124
 125     const video: VideoDetails = res.body
 126     for (const f of video.files) {
 127       checkMagnetWebseeds(f, webseeds, server)
 128     }
 129   }
 130 }
 131
 132 async function check2Webseeds (videoUUID?: string) {
 133   if (!videoUUID) videoUUID = video1Server2UUID
 134
 135   const webseeds = [
 136     `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
 137     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 138   ]
 139
 140   for (const server of servers) {
 141     const res = await getVideo(server.url, videoUUID)
 142
 143     const video: VideoDetails = res.body
 144
 145     for (const file of video.files) {
 146       checkMagnetWebseeds(file, webseeds, server)
 147
 148       await makeGetRequest({
 149         url: servers[0].url,
 150         statusCodeExpected: 200,
 151         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 152         contentType: null
 153       })
 154       await makeGetRequest({
 155         url: servers[1].url,
 156         statusCodeExpected: 200,
 157         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 158         contentType: null
 159       })
 160     }
 161   }
 162
 163   const directories = [
 164     'test' + servers[0].internalServerNumber + '/redundancy',
 165     'test' + servers[1].internalServerNumber + '/videos'
 166   ]
 167
 168   for (const directory of directories) {
 169     const files = await readdir(join(root(), directory))
 170     expect(files).to.have.length.at.least(4)
 171
 172     for (const resolution of [ 240, 360, 480, 720 ]) {
 173       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 174     }
 175   }
 176 }
 177
 178 async function check0PlaylistRedundancies (videoUUID?: string) {
 179   if (!videoUUID) videoUUID = video1Server2UUID
 180
 181   for (const server of servers) {
 182     // With token to avoid issues with video follow constraints
 183     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 184     const video: VideoDetails = res.body
 185
 186     expect(video.streamingPlaylists).to.be.an('array')
 187     expect(video.streamingPlaylists).to.have.lengthOf(1)
 188     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 189   }
 190 }
 191
 192 async function check1PlaylistRedundancies (videoUUID?: string) {
 193   if (!videoUUID) videoUUID = video1Server2UUID
 194
 195   for (const server of servers) {
 196     const res = await getVideo(server.url, videoUUID)
 197     const video: VideoDetails = res.body
 198
 199     expect(video.streamingPlaylists).to.have.lengthOf(1)
 200     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 201
 202     const redundancy = video.streamingPlaylists[0].redundancies[0]
 203
 204     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 205   }
 206
 207   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 208   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 209
 210   const res = await getVideo(servers[0].url, videoUUID)
 211   const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
 212
 213   for (const resolution of [ 240, 360, 480, 720 ]) {
 214     await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
 215   }
 216
 217   const directories = [
 218     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 219     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 220   ]
 221
 222   for (const directory of directories) {
 223     const files = await readdir(join(root(), directory, videoUUID))
 224     expect(files).to.have.length.at.least(4)
 225
 226     for (const resolution of [ 240, 360, 480, 720 ]) {
 227       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 228
 229       expect(files.find(f => f === filename)).to.not.be.undefined
 230     }
 231   }
 232 }
 233
 234 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
 235   let totalSize: number = null
 236   let statsLength = 1
 237
 238   if (strategy !== 'manual') {
 239     totalSize = 409600
 240     statsLength = 2
 241   }
 242
 243   const res = await getStats(servers[0].url)
 244   const data: ServerStats = res.body
 245
 246   expect(data.videosRedundancy).to.have.lengthOf(statsLength)
 247
 248   const stat = data.videosRedundancy[0]
 249   expect(stat.strategy).to.equal(strategy)
 250   expect(stat.totalSize).to.equal(totalSize)
 251
 252   return stat
 253 }
 254
 255 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) {
 256   const stat = await checkStatsGlobal(strategy)
 257
 258   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 259   expect(stat.totalVideoFiles).to.equal(4)
 260   expect(stat.totalVideos).to.equal(1)
 261 }
 262
 263 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) {
 264   const stat = await checkStatsGlobal(strategy)
 265
 266   expect(stat.totalUsed).to.equal(0)
 267   expect(stat.totalVideoFiles).to.equal(0)
 268   expect(stat.totalVideos).to.equal(0)
 269 }
 270
 271 async function enableRedundancyOnServer1 () {
 272   await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
 273
 274   const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' })
 275   const follows: ActorFollow[] = res.body.data
 276   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 277   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 278
 279   expect(server3).to.not.be.undefined
 280   expect(server3.following.hostRedundancyAllowed).to.be.false
 281
 282   expect(server2).to.not.be.undefined
 283   expect(server2.following.hostRedundancyAllowed).to.be.true
 284 }
 285
 286 async function disableRedundancyOnServer1 () {
 287   await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false)
 288
 289   const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' })
 290   const follows: ActorFollow[] = res.body.data
 291   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 292   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 293
 294   expect(server3).to.not.be.undefined
 295   expect(server3.following.hostRedundancyAllowed).to.be.false
 296
 297   expect(server2).to.not.be.undefined
 298   expect(server2.following.hostRedundancyAllowed).to.be.false
 299 }
 300
 301 describe('Test videos redundancy', function () {
 302
 303   describe('With most-views strategy', function () {
 304     const strategy = 'most-views'
 305
 306     before(function () {
 307       this.timeout(120000)
 308
 309       return flushAndRunServers(strategy)
 310     })
 311
 312     it('Should have 1 webseed on the first video', async function () {
 313       await check1WebSeed()
 314       await check0PlaylistRedundancies()
 315       await checkStatsWith1Webseed(strategy)
 316     })
 317
 318     it('Should enable redundancy on server 1', function () {
 319       return enableRedundancyOnServer1()
 320     })
 321
 322     it('Should have 2 webseeds on the first video', async function () {
 323       this.timeout(80000)
 324
 325       await waitJobs(servers)
 326       await waitUntilLog(servers[0], 'Duplicated ', 5)
 327       await waitJobs(servers)
 328
 329       await check2Webseeds()
 330       await check1PlaylistRedundancies()
 331       await checkStatsWith2Webseed(strategy)
 332     })
 333
 334     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 335       this.timeout(80000)
 336
 337       await disableRedundancyOnServer1()
 338
 339       await waitJobs(servers)
 340       await wait(5000)
 341
 342       await check1WebSeed()
 343       await check0PlaylistRedundancies()
 344
 345       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
 346     })
 347
 348     after(async function () {
 349       return cleanupTests(servers)
 350     })
 351   })
 352
 353   describe('With trending strategy', function () {
 354     const strategy = 'trending'
 355
 356     before(function () {
 357       this.timeout(120000)
 358
 359       return flushAndRunServers(strategy)
 360     })
 361
 362     it('Should have 1 webseed on the first video', async function () {
 363       await check1WebSeed()
 364       await check0PlaylistRedundancies()
 365       await checkStatsWith1Webseed(strategy)
 366     })
 367
 368     it('Should enable redundancy on server 1', function () {
 369       return enableRedundancyOnServer1()
 370     })
 371
 372     it('Should have 2 webseeds on the first video', async function () {
 373       this.timeout(80000)
 374
 375       await waitJobs(servers)
 376       await waitUntilLog(servers[0], 'Duplicated ', 5)
 377       await waitJobs(servers)
 378
 379       await check2Webseeds()
 380       await check1PlaylistRedundancies()
 381       await checkStatsWith2Webseed(strategy)
 382     })
 383
 384     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 385       this.timeout(80000)
 386
 387       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 388
 389       await waitJobs(servers)
 390       await wait(5000)
 391
 392       await check1WebSeed()
 393       await check0PlaylistRedundancies()
 394
 395       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
 396     })
 397
 398     after(async function () {
 399       await cleanupTests(servers)
 400     })
 401   })
 402
 403   describe('With recently added strategy', function () {
 404     const strategy = 'recently-added'
 405
 406     before(function () {
 407       this.timeout(120000)
 408
 409       return flushAndRunServers(strategy, { min_views: 3 })
 410     })
 411
 412     it('Should have 1 webseed on the first video', async function () {
 413       await check1WebSeed()
 414       await check0PlaylistRedundancies()
 415       await checkStatsWith1Webseed(strategy)
 416     })
 417
 418     it('Should enable redundancy on server 1', function () {
 419       return enableRedundancyOnServer1()
 420     })
 421
 422     it('Should still have 1 webseed on the first video', async function () {
 423       this.timeout(80000)
 424
 425       await waitJobs(servers)
 426       await wait(15000)
 427       await waitJobs(servers)
 428
 429       await check1WebSeed()
 430       await check0PlaylistRedundancies()
 431       await checkStatsWith1Webseed(strategy)
 432     })
 433
 434     it('Should view 2 times the first video to have > min_views config', async function () {
 435       this.timeout(80000)
 436
 437       await viewVideo(servers[0].url, video1Server2UUID)
 438       await viewVideo(servers[2].url, video1Server2UUID)
 439
 440       await wait(10000)
 441       await waitJobs(servers)
 442     })
 443
 444     it('Should have 2 webseeds on the first video', async function () {
 445       this.timeout(80000)
 446
 447       await waitJobs(servers)
 448       await waitUntilLog(servers[0], 'Duplicated ', 5)
 449       await waitJobs(servers)
 450
 451       await check2Webseeds()
 452       await check1PlaylistRedundancies()
 453       await checkStatsWith2Webseed(strategy)
 454     })
 455
 456     it('Should remove the video and the redundancy files', async function () {
 457       this.timeout(20000)
 458
 459       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 460
 461       await waitJobs(servers)
 462
 463       for (const server of servers) {
 464         await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
 465       }
 466     })
 467
 468     after(async function () {
 469       await cleanupTests(servers)
 470     })
 471   })
 472
 473   describe('With manual strategy', function () {
 474     before(function () {
 475       this.timeout(120000)
 476
 477       return flushAndRunServers(null)
 478     })
 479
 480     it('Should have 1 webseed on the first video', async function () {
 481       await check1WebSeed()
 482       await check0PlaylistRedundancies()
 483       await checkStatsWith1Webseed('manual')
 484     })
 485
 486     it('Should create a redundancy on first video', async function () {
 487       await addVideoRedundancy({
 488         url: servers[0].url,
 489         accessToken: servers[0].accessToken,
 490         videoId: video1Server2Id
 491       })
 492     })
 493
 494     it('Should have 2 webseeds on the first video', async function () {
 495       this.timeout(80000)
 496
 497       await waitJobs(servers)
 498       await waitUntilLog(servers[0], 'Duplicated ', 5)
 499       await waitJobs(servers)
 500
 501       await check2Webseeds()
 502       await check1PlaylistRedundancies()
 503       await checkStatsWith2Webseed('manual')
 504     })
 505
 506     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
 507       this.timeout(80000)
 508
 509       const res = await listVideoRedundancies({
 510         url: servers[0].url,
 511         accessToken: servers[0].accessToken,
 512         target: 'remote-videos'
 513       })
 514
 515       const videos = res.body.data as VideoRedundancy[]
 516       expect(videos).to.have.lengthOf(1)
 517
 518       const video = videos[0]
 519       for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
 520         await removeVideoRedundancy({
 521           url: servers[0].url,
 522           accessToken: servers[0].accessToken,
 523           redundancyId: r.id
 524         })
 525       }
 526
 527       await waitJobs(servers)
 528       await wait(5000)
 529
 530       await check1WebSeed()
 531       await check0PlaylistRedundancies()
 532
 533       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 534     })
 535
 536     after(async function () {
 537       await cleanupTests(servers)
 538     })
 539   })
 540
 541   describe('Test expiration', function () {
 542     const strategy = 'recently-added'
 543
 544     async function checkContains (servers: ServerInfo[], str: string) {
 545       for (const server of servers) {
 546         const res = await getVideo(server.url, video1Server2UUID)
 547         const video: VideoDetails = res.body
 548
 549         for (const f of video.files) {
 550           expect(f.magnetUri).to.contain(str)
 551         }
 552       }
 553     }
 554
 555     async function checkNotContains (servers: ServerInfo[], str: string) {
 556       for (const server of servers) {
 557         const res = await getVideo(server.url, video1Server2UUID)
 558         const video: VideoDetails = res.body
 559
 560         for (const f of video.files) {
 561           expect(f.magnetUri).to.not.contain(str)
 562         }
 563       }
 564     }
 565
 566     before(async function () {
 567       this.timeout(120000)
 568
 569       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 570
 571       await enableRedundancyOnServer1()
 572     })
 573
 574     it('Should still have 2 webseeds after 10 seconds', async function () {
 575       this.timeout(80000)
 576
 577       await wait(10000)
 578
 579       try {
 580         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 581       } catch {
 582         // Maybe a server deleted a redundancy in the scheduler
 583         await wait(2000)
 584
 585         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 586       }
 587     })
 588
 589     it('Should stop server 1 and expire video redundancy', async function () {
 590       this.timeout(80000)
 591
 592       killallServers([ servers[0] ])
 593
 594       await wait(15000)
 595
 596       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 597     })
 598
 599     after(async function () {
 600       await cleanupTests(servers)
 601     })
 602   })
 603
 604   describe('Test file replacement', function () {
 605     let video2Server2UUID: string
 606     const strategy = 'recently-added'
 607
 608     before(async function () {
 609       this.timeout(120000)
 610
 611       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 612
 613       await enableRedundancyOnServer1()
 614
 615       await waitJobs(servers)
 616       await waitUntilLog(servers[0], 'Duplicated ', 5)
 617       await waitJobs(servers)
 618
 619       await check2Webseeds()
 620       await check1PlaylistRedundancies()
 621       await checkStatsWith2Webseed(strategy)
 622
 623       const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' })
 624       video2Server2UUID = res.body.video.uuid
 625     })
 626
 627     it('Should cache video 2 webseeds on the first video', async function () {
 628       this.timeout(120000)
 629
 630       await waitJobs(servers)
 631
 632       let checked = false
 633
 634       while (checked === false) {
 635         await wait(1000)
 636
 637         try {
 638           await check1WebSeed(video1Server2UUID)
 639           await check0PlaylistRedundancies(video1Server2UUID)
 640           await check2Webseeds(video2Server2UUID)
 641           await check1PlaylistRedundancies(video2Server2UUID)
 642
 643           checked = true
 644         } catch {
 645           checked = false
 646         }
 647       }
 648     })
 649
 650     it('Should disable strategy and remove redundancies', async function () {
 651       this.timeout(80000)
 652
 653       await waitJobs(servers)
 654
 655       killallServers([ servers[0] ])
 656       await reRunServer(servers[0], {
 657         redundancy: {
 658           videos: {
 659             check_interval: '1 second',
 660             strategies: []
 661           }
 662         }
 663       })
 664
 665       await waitJobs(servers)
 666
 667       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
 668     })
 669
 670     after(async function () {
 671       await cleanupTests(servers)
 672     })
 673   })
 674 })