server/tests/api/redundancy/redundancy.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   checkSegmentHash,
   8   checkVideoFilesWereRemoved,
   9   cleanupTests,
  10   doubleFollow,
  11   flushAndRunMultipleServers,
  12   getFollowingListPaginationAndSort,
  13   getVideo,
  14   getVideoWithToken,
  15   immutableAssign,
  16   killallServers,
  17   makeGetRequest,
  18   removeVideo,
  19   reRunServer,
  20   root,
  21   ServerInfo,
  22   setAccessTokensToServers,
  23   unfollow,
  24   uploadVideo,
  25   viewVideo,
  26   wait,
  27   waitUntilLog
  28 } from '../../../../shared/extra-utils'
  29 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
  30
  31 import * as magnetUtil from 'magnet-uri'
  32 import {
  33   addVideoRedundancy,
  34   listVideoRedundancies,
  35   removeVideoRedundancy,
  36   updateRedundancy
  37 } from '../../../../shared/extra-utils/server/redundancy'
  38 import { ActorFollow } from '../../../../shared/models/actors'
  39 import { readdir } from 'fs-extra'
  40 import { join } from 'path'
  41 import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
  42 import { getStats } from '../../../../shared/extra-utils/server/stats'
  43 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  44 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
  45
  46 const expect = chai.expect
  47
  48 let servers: ServerInfo[] = []
  49 let video1Server2UUID: string
  50 let video1Server2Id: number
  51
  52 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  53   const parsed = magnetUtil.decode(file.magnetUri)
  54
  55   for (const ws of baseWebseeds) {
  56     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  57     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  58   }
  59
  60   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  61 }
  62
  63 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) {
  64   const strategies: any[] = []
  65
  66   if (strategy !== null) {
  67     strategies.push(
  68       immutableAssign({
  69         min_lifetime: '1 hour',
  70         strategy: strategy,
  71         size: '400KB'
  72       }, additionalParams)
  73     )
  74   }
  75
  76   const config = {
  77     transcoding: {
  78       hls: {
  79         enabled: true
  80       }
  81     },
  82     redundancy: {
  83       videos: {
  84         check_interval: '5 seconds',
  85         strategies
  86       }
  87     }
  88   }
  89
  90   servers = await flushAndRunMultipleServers(3, config)
  91
  92   // Get the access tokens
  93   await setAccessTokensToServers(servers)
  94
  95   {
  96     const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
  97     video1Server2UUID = res.body.video.uuid
  98     video1Server2Id = res.body.video.id
  99
 100     await viewVideo(servers[1].url, video1Server2UUID)
 101   }
 102
 103   await waitJobs(servers)
 104
 105   // Server 1 and server 2 follow each other
 106   await doubleFollow(servers[0], servers[1])
 107   // Server 1 and server 3 follow each other
 108   await doubleFollow(servers[0], servers[2])
 109   // Server 2 and server 3 follow each other
 110   await doubleFollow(servers[1], servers[2])
 111
 112   await waitJobs(servers)
 113 }
 114
 115 async function check1WebSeed (videoUUID?: string) {
 116   if (!videoUUID) videoUUID = video1Server2UUID
 117
 118   const webseeds = [
 119     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 120   ]
 121
 122   for (const server of servers) {
 123     // With token to avoid issues with video follow constraints
 124     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 125
 126     const video: VideoDetails = res.body
 127     for (const f of video.files) {
 128       checkMagnetWebseeds(f, webseeds, server)
 129     }
 130   }
 131 }
 132
 133 async function check2Webseeds (videoUUID?: string) {
 134   if (!videoUUID) videoUUID = video1Server2UUID
 135
 136   const webseeds = [
 137     `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
 138     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 139   ]
 140
 141   for (const server of servers) {
 142     const res = await getVideo(server.url, videoUUID)
 143
 144     const video: VideoDetails = res.body
 145
 146     for (const file of video.files) {
 147       checkMagnetWebseeds(file, webseeds, server)
 148
 149       await makeGetRequest({
 150         url: servers[0].url,
 151         statusCodeExpected: HttpStatusCode.OK_200,
 152         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 153         contentType: null
 154       })
 155       await makeGetRequest({
 156         url: servers[1].url,
 157         statusCodeExpected: HttpStatusCode.OK_200,
 158         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 159         contentType: null
 160       })
 161     }
 162   }
 163
 164   const directories = [
 165     'test' + servers[0].internalServerNumber + '/redundancy',
 166     'test' + servers[1].internalServerNumber + '/videos'
 167   ]
 168
 169   for (const directory of directories) {
 170     const files = await readdir(join(root(), directory))
 171     expect(files).to.have.length.at.least(4)
 172
 173     for (const resolution of [ 240, 360, 480, 720 ]) {
 174       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 175     }
 176   }
 177 }
 178
 179 async function check0PlaylistRedundancies (videoUUID?: string) {
 180   if (!videoUUID) videoUUID = video1Server2UUID
 181
 182   for (const server of servers) {
 183     // With token to avoid issues with video follow constraints
 184     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 185     const video: VideoDetails = res.body
 186
 187     expect(video.streamingPlaylists).to.be.an('array')
 188     expect(video.streamingPlaylists).to.have.lengthOf(1)
 189     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 190   }
 191 }
 192
 193 async function check1PlaylistRedundancies (videoUUID?: string) {
 194   if (!videoUUID) videoUUID = video1Server2UUID
 195
 196   for (const server of servers) {
 197     const res = await getVideo(server.url, videoUUID)
 198     const video: VideoDetails = res.body
 199
 200     expect(video.streamingPlaylists).to.have.lengthOf(1)
 201     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 202
 203     const redundancy = video.streamingPlaylists[0].redundancies[0]
 204
 205     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 206   }
 207
 208   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 209   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 210
 211   const res = await getVideo(servers[0].url, videoUUID)
 212   const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
 213
 214   for (const resolution of [ 240, 360, 480, 720 ]) {
 215     await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
 216   }
 217
 218   const directories = [
 219     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 220     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 221   ]
 222
 223   for (const directory of directories) {
 224     const files = await readdir(join(root(), directory, videoUUID))
 225     expect(files).to.have.length.at.least(4)
 226
 227     for (const resolution of [ 240, 360, 480, 720 ]) {
 228       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 229
 230       expect(files.find(f => f === filename)).to.not.be.undefined
 231     }
 232   }
 233 }
 234
 235 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
 236   let totalSize: number = null
 237   let statsLength = 1
 238
 239   if (strategy !== 'manual') {
 240     totalSize = 409600
 241     statsLength = 2
 242   }
 243
 244   const res = await getStats(servers[0].url)
 245   const data: ServerStats = res.body
 246
 247   expect(data.videosRedundancy).to.have.lengthOf(statsLength)
 248
 249   const stat = data.videosRedundancy[0]
 250   expect(stat.strategy).to.equal(strategy)
 251   expect(stat.totalSize).to.equal(totalSize)
 252
 253   return stat
 254 }
 255
 256 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) {
 257   const stat = await checkStatsGlobal(strategy)
 258
 259   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 260   expect(stat.totalVideoFiles).to.equal(4)
 261   expect(stat.totalVideos).to.equal(1)
 262 }
 263
 264 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) {
 265   const stat = await checkStatsGlobal(strategy)
 266
 267   expect(stat.totalUsed).to.equal(0)
 268   expect(stat.totalVideoFiles).to.equal(0)
 269   expect(stat.totalVideos).to.equal(0)
 270 }
 271
 272 async function enableRedundancyOnServer1 () {
 273   await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
 274
 275   const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' })
 276   const follows: ActorFollow[] = res.body.data
 277   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 278   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 279
 280   expect(server3).to.not.be.undefined
 281   expect(server3.following.hostRedundancyAllowed).to.be.false
 282
 283   expect(server2).to.not.be.undefined
 284   expect(server2.following.hostRedundancyAllowed).to.be.true
 285 }
 286
 287 async function disableRedundancyOnServer1 () {
 288   await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false)
 289
 290   const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' })
 291   const follows: ActorFollow[] = res.body.data
 292   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 293   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 294
 295   expect(server3).to.not.be.undefined
 296   expect(server3.following.hostRedundancyAllowed).to.be.false
 297
 298   expect(server2).to.not.be.undefined
 299   expect(server2.following.hostRedundancyAllowed).to.be.false
 300 }
 301
 302 describe('Test videos redundancy', function () {
 303
 304   describe('With most-views strategy', function () {
 305     const strategy = 'most-views'
 306
 307     before(function () {
 308       this.timeout(120000)
 309
 310       return flushAndRunServers(strategy)
 311     })
 312
 313     it('Should have 1 webseed on the first video', async function () {
 314       await check1WebSeed()
 315       await check0PlaylistRedundancies()
 316       await checkStatsWith1Webseed(strategy)
 317     })
 318
 319     it('Should enable redundancy on server 1', function () {
 320       return enableRedundancyOnServer1()
 321     })
 322
 323     it('Should have 2 webseeds on the first video', async function () {
 324       this.timeout(80000)
 325
 326       await waitJobs(servers)
 327       await waitUntilLog(servers[0], 'Duplicated ', 5)
 328       await waitJobs(servers)
 329
 330       await check2Webseeds()
 331       await check1PlaylistRedundancies()
 332       await checkStatsWith2Webseed(strategy)
 333     })
 334
 335     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 336       this.timeout(80000)
 337
 338       await disableRedundancyOnServer1()
 339
 340       await waitJobs(servers)
 341       await wait(5000)
 342
 343       await check1WebSeed()
 344       await check0PlaylistRedundancies()
 345
 346       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
 347     })
 348
 349     after(async function () {
 350       return cleanupTests(servers)
 351     })
 352   })
 353
 354   describe('With trending strategy', function () {
 355     const strategy = 'trending'
 356
 357     before(function () {
 358       this.timeout(120000)
 359
 360       return flushAndRunServers(strategy)
 361     })
 362
 363     it('Should have 1 webseed on the first video', async function () {
 364       await check1WebSeed()
 365       await check0PlaylistRedundancies()
 366       await checkStatsWith1Webseed(strategy)
 367     })
 368
 369     it('Should enable redundancy on server 1', function () {
 370       return enableRedundancyOnServer1()
 371     })
 372
 373     it('Should have 2 webseeds on the first video', async function () {
 374       this.timeout(80000)
 375
 376       await waitJobs(servers)
 377       await waitUntilLog(servers[0], 'Duplicated ', 5)
 378       await waitJobs(servers)
 379
 380       await check2Webseeds()
 381       await check1PlaylistRedundancies()
 382       await checkStatsWith2Webseed(strategy)
 383     })
 384
 385     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 386       this.timeout(80000)
 387
 388       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 389
 390       await waitJobs(servers)
 391       await wait(5000)
 392
 393       await check1WebSeed()
 394       await check0PlaylistRedundancies()
 395
 396       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
 397     })
 398
 399     after(async function () {
 400       await cleanupTests(servers)
 401     })
 402   })
 403
 404   describe('With recently added strategy', function () {
 405     const strategy = 'recently-added'
 406
 407     before(function () {
 408       this.timeout(120000)
 409
 410       return flushAndRunServers(strategy, { min_views: 3 })
 411     })
 412
 413     it('Should have 1 webseed on the first video', async function () {
 414       await check1WebSeed()
 415       await check0PlaylistRedundancies()
 416       await checkStatsWith1Webseed(strategy)
 417     })
 418
 419     it('Should enable redundancy on server 1', function () {
 420       return enableRedundancyOnServer1()
 421     })
 422
 423     it('Should still have 1 webseed on the first video', async function () {
 424       this.timeout(80000)
 425
 426       await waitJobs(servers)
 427       await wait(15000)
 428       await waitJobs(servers)
 429
 430       await check1WebSeed()
 431       await check0PlaylistRedundancies()
 432       await checkStatsWith1Webseed(strategy)
 433     })
 434
 435     it('Should view 2 times the first video to have > min_views config', async function () {
 436       this.timeout(80000)
 437
 438       await viewVideo(servers[0].url, video1Server2UUID)
 439       await viewVideo(servers[2].url, video1Server2UUID)
 440
 441       await wait(10000)
 442       await waitJobs(servers)
 443     })
 444
 445     it('Should have 2 webseeds on the first video', async function () {
 446       this.timeout(80000)
 447
 448       await waitJobs(servers)
 449       await waitUntilLog(servers[0], 'Duplicated ', 5)
 450       await waitJobs(servers)
 451
 452       await check2Webseeds()
 453       await check1PlaylistRedundancies()
 454       await checkStatsWith2Webseed(strategy)
 455     })
 456
 457     it('Should remove the video and the redundancy files', async function () {
 458       this.timeout(20000)
 459
 460       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 461
 462       await waitJobs(servers)
 463
 464       for (const server of servers) {
 465         await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
 466       }
 467     })
 468
 469     after(async function () {
 470       await cleanupTests(servers)
 471     })
 472   })
 473
 474   describe('With manual strategy', function () {
 475     before(function () {
 476       this.timeout(120000)
 477
 478       return flushAndRunServers(null)
 479     })
 480
 481     it('Should have 1 webseed on the first video', async function () {
 482       await check1WebSeed()
 483       await check0PlaylistRedundancies()
 484       await checkStatsWith1Webseed('manual')
 485     })
 486
 487     it('Should create a redundancy on first video', async function () {
 488       await addVideoRedundancy({
 489         url: servers[0].url,
 490         accessToken: servers[0].accessToken,
 491         videoId: video1Server2Id
 492       })
 493     })
 494
 495     it('Should have 2 webseeds on the first video', async function () {
 496       this.timeout(80000)
 497
 498       await waitJobs(servers)
 499       await waitUntilLog(servers[0], 'Duplicated ', 5)
 500       await waitJobs(servers)
 501
 502       await check2Webseeds()
 503       await check1PlaylistRedundancies()
 504       await checkStatsWith2Webseed('manual')
 505     })
 506
 507     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
 508       this.timeout(80000)
 509
 510       const res = await listVideoRedundancies({
 511         url: servers[0].url,
 512         accessToken: servers[0].accessToken,
 513         target: 'remote-videos'
 514       })
 515
 516       const videos = res.body.data as VideoRedundancy[]
 517       expect(videos).to.have.lengthOf(1)
 518
 519       const video = videos[0]
 520       for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
 521         await removeVideoRedundancy({
 522           url: servers[0].url,
 523           accessToken: servers[0].accessToken,
 524           redundancyId: r.id
 525         })
 526       }
 527
 528       await waitJobs(servers)
 529       await wait(5000)
 530
 531       await check1WebSeed()
 532       await check0PlaylistRedundancies()
 533
 534       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 535     })
 536
 537     after(async function () {
 538       await cleanupTests(servers)
 539     })
 540   })
 541
 542   describe('Test expiration', function () {
 543     const strategy = 'recently-added'
 544
 545     async function checkContains (servers: ServerInfo[], str: string) {
 546       for (const server of servers) {
 547         const res = await getVideo(server.url, video1Server2UUID)
 548         const video: VideoDetails = res.body
 549
 550         for (const f of video.files) {
 551           expect(f.magnetUri).to.contain(str)
 552         }
 553       }
 554     }
 555
 556     async function checkNotContains (servers: ServerInfo[], str: string) {
 557       for (const server of servers) {
 558         const res = await getVideo(server.url, video1Server2UUID)
 559         const video: VideoDetails = res.body
 560
 561         for (const f of video.files) {
 562           expect(f.magnetUri).to.not.contain(str)
 563         }
 564       }
 565     }
 566
 567     before(async function () {
 568       this.timeout(120000)
 569
 570       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 571
 572       await enableRedundancyOnServer1()
 573     })
 574
 575     it('Should still have 2 webseeds after 10 seconds', async function () {
 576       this.timeout(80000)
 577
 578       await wait(10000)
 579
 580       try {
 581         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 582       } catch {
 583         // Maybe a server deleted a redundancy in the scheduler
 584         await wait(2000)
 585
 586         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 587       }
 588     })
 589
 590     it('Should stop server 1 and expire video redundancy', async function () {
 591       this.timeout(80000)
 592
 593       killallServers([ servers[0] ])
 594
 595       await wait(15000)
 596
 597       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 598     })
 599
 600     after(async function () {
 601       await cleanupTests(servers)
 602     })
 603   })
 604
 605   describe('Test file replacement', function () {
 606     let video2Server2UUID: string
 607     const strategy = 'recently-added'
 608
 609     before(async function () {
 610       this.timeout(120000)
 611
 612       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 613
 614       await enableRedundancyOnServer1()
 615
 616       await waitJobs(servers)
 617       await waitUntilLog(servers[0], 'Duplicated ', 5)
 618       await waitJobs(servers)
 619
 620       await check2Webseeds()
 621       await check1PlaylistRedundancies()
 622       await checkStatsWith2Webseed(strategy)
 623
 624       const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' })
 625       video2Server2UUID = res.body.video.uuid
 626     })
 627
 628     it('Should cache video 2 webseeds on the first video', async function () {
 629       this.timeout(120000)
 630
 631       await waitJobs(servers)
 632
 633       let checked = false
 634
 635       while (checked === false) {
 636         await wait(1000)
 637
 638         try {
 639           await check1WebSeed(video1Server2UUID)
 640           await check0PlaylistRedundancies(video1Server2UUID)
 641           await check2Webseeds(video2Server2UUID)
 642           await check1PlaylistRedundancies(video2Server2UUID)
 643
 644           checked = true
 645         } catch {
 646           checked = false
 647         }
 648       }
 649     })
 650
 651     it('Should disable strategy and remove redundancies', async function () {
 652       this.timeout(80000)
 653
 654       await waitJobs(servers)
 655
 656       killallServers([ servers[0] ])
 657       await reRunServer(servers[0], {
 658         redundancy: {
 659           videos: {
 660             check_interval: '1 second',
 661             strategies: []
 662           }
 663         }
 664       })
 665
 666       await waitJobs(servers)
 667
 668       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
 669     })
 670
 671     after(async function () {
 672       await cleanupTests(servers)
 673     })
 674   })
 675 })