server/tests/api/redundancy/redundancy.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { readdir } from 'fs-extra'
   6 import * as magnetUtil from 'magnet-uri'
   7 import { join } from 'path'
   8 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
   9 import {
  10   checkSegmentHash,
  11   checkVideoFilesWereRemoved,
  12   cleanupTests,
  13   doubleFollow,
  14   flushAndRunMultipleServers,
  15   getVideo,
  16   getVideoWithToken,
  17   immutableAssign,
  18   killallServers,
  19   makeGetRequest,
  20   removeVideo,
  21   reRunServer,
  22   root,
  23   ServerInfo,
  24   setAccessTokensToServers,
  25   updateVideo,
  26   uploadVideo,
  27   viewVideo,
  28   wait,
  29   waitUntilLog
  30 } from '../../../../shared/extra-utils'
  31 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
  32 import {
  33   addVideoRedundancy,
  34   listVideoRedundancies,
  35   removeVideoRedundancy,
  36   updateRedundancy
  37 } from '../../../../shared/extra-utils/server/redundancy'
  38 import { getStats } from '../../../../shared/extra-utils/server/stats'
  39 import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
  40 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  41 import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos'
  42
  43 const expect = chai.expect
  44
  45 let servers: ServerInfo[] = []
  46 let video1Server2UUID: string
  47 let video1Server2Id: number
  48
  49 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  50   const parsed = magnetUtil.decode(file.magnetUri)
  51
  52   for (const ws of baseWebseeds) {
  53     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  54     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  55   }
  56
  57   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  58 }
  59
  60 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
  61   const strategies: any[] = []
  62
  63   if (strategy !== null) {
  64     strategies.push(
  65       immutableAssign({
  66         min_lifetime: '1 hour',
  67         strategy: strategy,
  68         size: '400KB'
  69       }, additionalParams)
  70     )
  71   }
  72
  73   const config = {
  74     transcoding: {
  75       webtorrent: {
  76         enabled: withWebtorrent
  77       },
  78       hls: {
  79         enabled: true
  80       }
  81     },
  82     redundancy: {
  83       videos: {
  84         check_interval: '5 seconds',
  85         strategies
  86       }
  87     }
  88   }
  89
  90   servers = await flushAndRunMultipleServers(3, config)
  91
  92   // Get the access tokens
  93   await setAccessTokensToServers(servers)
  94
  95   {
  96     const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
  97     video1Server2UUID = res.body.video.uuid
  98     video1Server2Id = res.body.video.id
  99
 100     await viewVideo(servers[1].url, video1Server2UUID)
 101   }
 102
 103   await waitJobs(servers)
 104
 105   // Server 1 and server 2 follow each other
 106   await doubleFollow(servers[0], servers[1])
 107   // Server 1 and server 3 follow each other
 108   await doubleFollow(servers[0], servers[2])
 109   // Server 2 and server 3 follow each other
 110   await doubleFollow(servers[1], servers[2])
 111
 112   await waitJobs(servers)
 113 }
 114
 115 async function check1WebSeed (videoUUID?: string) {
 116   if (!videoUUID) videoUUID = video1Server2UUID
 117
 118   const webseeds = [
 119     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 120   ]
 121
 122   for (const server of servers) {
 123     // With token to avoid issues with video follow constraints
 124     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 125
 126     const video: VideoDetails = res.body
 127     for (const f of video.files) {
 128       checkMagnetWebseeds(f, webseeds, server)
 129     }
 130   }
 131 }
 132
 133 async function check2Webseeds (videoUUID?: string) {
 134   if (!videoUUID) videoUUID = video1Server2UUID
 135
 136   const webseeds = [
 137     `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
 138     `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
 139   ]
 140
 141   for (const server of servers) {
 142     const res = await getVideo(server.url, videoUUID)
 143
 144     const video: VideoDetails = res.body
 145
 146     for (const file of video.files) {
 147       checkMagnetWebseeds(file, webseeds, server)
 148
 149       await makeGetRequest({
 150         url: servers[0].url,
 151         statusCodeExpected: HttpStatusCode.OK_200,
 152         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 153         contentType: null
 154       })
 155       await makeGetRequest({
 156         url: servers[1].url,
 157         statusCodeExpected: HttpStatusCode.OK_200,
 158         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 159         contentType: null
 160       })
 161     }
 162   }
 163
 164   const directories = [
 165     'test' + servers[0].internalServerNumber + '/redundancy',
 166     'test' + servers[1].internalServerNumber + '/videos'
 167   ]
 168
 169   for (const directory of directories) {
 170     const files = await readdir(join(root(), directory))
 171     expect(files).to.have.length.at.least(4)
 172
 173     for (const resolution of [ 240, 360, 480, 720 ]) {
 174       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 175     }
 176   }
 177 }
 178
 179 async function check0PlaylistRedundancies (videoUUID?: string) {
 180   if (!videoUUID) videoUUID = video1Server2UUID
 181
 182   for (const server of servers) {
 183     // With token to avoid issues with video follow constraints
 184     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 185     const video: VideoDetails = res.body
 186
 187     expect(video.streamingPlaylists).to.be.an('array')
 188     expect(video.streamingPlaylists).to.have.lengthOf(1)
 189     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 190   }
 191 }
 192
 193 async function check1PlaylistRedundancies (videoUUID?: string) {
 194   if (!videoUUID) videoUUID = video1Server2UUID
 195
 196   for (const server of servers) {
 197     const res = await getVideo(server.url, videoUUID)
 198     const video: VideoDetails = res.body
 199
 200     expect(video.streamingPlaylists).to.have.lengthOf(1)
 201     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 202
 203     const redundancy = video.streamingPlaylists[0].redundancies[0]
 204
 205     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 206   }
 207
 208   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 209   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 210
 211   const res = await getVideo(servers[0].url, videoUUID)
 212   const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
 213
 214   for (const resolution of [ 240, 360, 480, 720 ]) {
 215     await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
 216   }
 217
 218   const directories = [
 219     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 220     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 221   ]
 222
 223   for (const directory of directories) {
 224     const files = await readdir(join(root(), directory, videoUUID))
 225     expect(files).to.have.length.at.least(4)
 226
 227     for (const resolution of [ 240, 360, 480, 720 ]) {
 228       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 229
 230       expect(files.find(f => f === filename)).to.not.be.undefined
 231     }
 232   }
 233 }
 234
 235 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
 236   let totalSize: number = null
 237   let statsLength = 1
 238
 239   if (strategy !== 'manual') {
 240     totalSize = 409600
 241     statsLength = 2
 242   }
 243
 244   const res = await getStats(servers[0].url)
 245   const data: ServerStats = res.body
 246
 247   expect(data.videosRedundancy).to.have.lengthOf(statsLength)
 248
 249   const stat = data.videosRedundancy[0]
 250   expect(stat.strategy).to.equal(strategy)
 251   expect(stat.totalSize).to.equal(totalSize)
 252
 253   return stat
 254 }
 255
 256 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
 257   const stat = await checkStatsGlobal(strategy)
 258
 259   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 260   expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
 261   expect(stat.totalVideos).to.equal(1)
 262 }
 263
 264 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
 265   const stat = await checkStatsGlobal(strategy)
 266
 267   expect(stat.totalUsed).to.equal(0)
 268   expect(stat.totalVideoFiles).to.equal(0)
 269   expect(stat.totalVideos).to.equal(0)
 270 }
 271
 272 async function findServerFollows () {
 273   const body = await servers[0].followsCommand.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
 274   const follows = body.data
 275   const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
 276   const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
 277
 278   return { server2, server3 }
 279 }
 280
 281 async function enableRedundancyOnServer1 () {
 282   await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
 283
 284   const { server2, server3 } = await findServerFollows()
 285
 286   expect(server3).to.not.be.undefined
 287   expect(server3.following.hostRedundancyAllowed).to.be.false
 288
 289   expect(server2).to.not.be.undefined
 290   expect(server2.following.hostRedundancyAllowed).to.be.true
 291 }
 292
 293 async function disableRedundancyOnServer1 () {
 294   await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false)
 295
 296   const { server2, server3 } = await findServerFollows()
 297
 298   expect(server3).to.not.be.undefined
 299   expect(server3.following.hostRedundancyAllowed).to.be.false
 300
 301   expect(server2).to.not.be.undefined
 302   expect(server2.following.hostRedundancyAllowed).to.be.false
 303 }
 304
 305 describe('Test videos redundancy', function () {
 306
 307   describe('With most-views strategy', function () {
 308     const strategy = 'most-views'
 309
 310     before(function () {
 311       this.timeout(120000)
 312
 313       return flushAndRunServers(strategy)
 314     })
 315
 316     it('Should have 1 webseed on the first video', async function () {
 317       await check1WebSeed()
 318       await check0PlaylistRedundancies()
 319       await checkStatsWithoutRedundancy(strategy)
 320     })
 321
 322     it('Should enable redundancy on server 1', function () {
 323       return enableRedundancyOnServer1()
 324     })
 325
 326     it('Should have 2 webseeds on the first video', async function () {
 327       this.timeout(80000)
 328
 329       await waitJobs(servers)
 330       await waitUntilLog(servers[0], 'Duplicated ', 5)
 331       await waitJobs(servers)
 332
 333       await check2Webseeds()
 334       await check1PlaylistRedundancies()
 335       await checkStatsWith1Redundancy(strategy)
 336     })
 337
 338     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 339       this.timeout(80000)
 340
 341       await disableRedundancyOnServer1()
 342
 343       await waitJobs(servers)
 344       await wait(5000)
 345
 346       await check1WebSeed()
 347       await check0PlaylistRedundancies()
 348
 349       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
 350     })
 351
 352     after(async function () {
 353       return cleanupTests(servers)
 354     })
 355   })
 356
 357   describe('With trending strategy', function () {
 358     const strategy = 'trending'
 359
 360     before(function () {
 361       this.timeout(120000)
 362
 363       return flushAndRunServers(strategy)
 364     })
 365
 366     it('Should have 1 webseed on the first video', async function () {
 367       await check1WebSeed()
 368       await check0PlaylistRedundancies()
 369       await checkStatsWithoutRedundancy(strategy)
 370     })
 371
 372     it('Should enable redundancy on server 1', function () {
 373       return enableRedundancyOnServer1()
 374     })
 375
 376     it('Should have 2 webseeds on the first video', async function () {
 377       this.timeout(80000)
 378
 379       await waitJobs(servers)
 380       await waitUntilLog(servers[0], 'Duplicated ', 5)
 381       await waitJobs(servers)
 382
 383       await check2Webseeds()
 384       await check1PlaylistRedundancies()
 385       await checkStatsWith1Redundancy(strategy)
 386     })
 387
 388     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 389       this.timeout(80000)
 390
 391       await servers[0].followsCommand.unfollow({ target: servers[1] })
 392
 393       await waitJobs(servers)
 394       await wait(5000)
 395
 396       await check1WebSeed()
 397       await check0PlaylistRedundancies()
 398
 399       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
 400     })
 401
 402     after(async function () {
 403       await cleanupTests(servers)
 404     })
 405   })
 406
 407   describe('With recently added strategy', function () {
 408     const strategy = 'recently-added'
 409
 410     before(function () {
 411       this.timeout(120000)
 412
 413       return flushAndRunServers(strategy, { min_views: 3 })
 414     })
 415
 416     it('Should have 1 webseed on the first video', async function () {
 417       await check1WebSeed()
 418       await check0PlaylistRedundancies()
 419       await checkStatsWithoutRedundancy(strategy)
 420     })
 421
 422     it('Should enable redundancy on server 1', function () {
 423       return enableRedundancyOnServer1()
 424     })
 425
 426     it('Should still have 1 webseed on the first video', async function () {
 427       this.timeout(80000)
 428
 429       await waitJobs(servers)
 430       await wait(15000)
 431       await waitJobs(servers)
 432
 433       await check1WebSeed()
 434       await check0PlaylistRedundancies()
 435       await checkStatsWithoutRedundancy(strategy)
 436     })
 437
 438     it('Should view 2 times the first video to have > min_views config', async function () {
 439       this.timeout(80000)
 440
 441       await viewVideo(servers[0].url, video1Server2UUID)
 442       await viewVideo(servers[2].url, video1Server2UUID)
 443
 444       await wait(10000)
 445       await waitJobs(servers)
 446     })
 447
 448     it('Should have 2 webseeds on the first video', async function () {
 449       this.timeout(80000)
 450
 451       await waitJobs(servers)
 452       await waitUntilLog(servers[0], 'Duplicated ', 5)
 453       await waitJobs(servers)
 454
 455       await check2Webseeds()
 456       await check1PlaylistRedundancies()
 457       await checkStatsWith1Redundancy(strategy)
 458     })
 459
 460     it('Should remove the video and the redundancy files', async function () {
 461       this.timeout(20000)
 462
 463       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 464
 465       await waitJobs(servers)
 466
 467       for (const server of servers) {
 468         await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
 469       }
 470     })
 471
 472     after(async function () {
 473       await cleanupTests(servers)
 474     })
 475   })
 476
 477   describe('With only HLS files', function () {
 478     const strategy = 'recently-added'
 479
 480     before(async function () {
 481       this.timeout(120000)
 482
 483       await flushAndRunServers(strategy, { min_views: 3 }, false)
 484     })
 485
 486     it('Should have 0 playlist redundancy on the first video', async function () {
 487       await check1WebSeed()
 488       await check0PlaylistRedundancies()
 489     })
 490
 491     it('Should enable redundancy on server 1', function () {
 492       return enableRedundancyOnServer1()
 493     })
 494
 495     it('Should still have 0 redundancy on the first video', async function () {
 496       this.timeout(80000)
 497
 498       await waitJobs(servers)
 499       await wait(15000)
 500       await waitJobs(servers)
 501
 502       await check0PlaylistRedundancies()
 503       await checkStatsWithoutRedundancy(strategy)
 504     })
 505
 506     it('Should have 1 redundancy on the first video', async function () {
 507       this.timeout(160000)
 508
 509       await viewVideo(servers[0].url, video1Server2UUID)
 510       await viewVideo(servers[2].url, video1Server2UUID)
 511
 512       await wait(10000)
 513       await waitJobs(servers)
 514
 515       await waitJobs(servers)
 516       await waitUntilLog(servers[0], 'Duplicated ', 1)
 517       await waitJobs(servers)
 518
 519       await check1PlaylistRedundancies()
 520       await checkStatsWith1Redundancy(strategy, true)
 521     })
 522
 523     it('Should remove the video and the redundancy files', async function () {
 524       this.timeout(20000)
 525
 526       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 527
 528       await waitJobs(servers)
 529
 530       for (const server of servers) {
 531         await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
 532       }
 533     })
 534
 535     after(async function () {
 536       await cleanupTests(servers)
 537     })
 538   })
 539
 540   describe('With manual strategy', function () {
 541     before(function () {
 542       this.timeout(120000)
 543
 544       return flushAndRunServers(null)
 545     })
 546
 547     it('Should have 1 webseed on the first video', async function () {
 548       await check1WebSeed()
 549       await check0PlaylistRedundancies()
 550       await checkStatsWithoutRedundancy('manual')
 551     })
 552
 553     it('Should create a redundancy on first video', async function () {
 554       await addVideoRedundancy({
 555         url: servers[0].url,
 556         accessToken: servers[0].accessToken,
 557         videoId: video1Server2Id
 558       })
 559     })
 560
 561     it('Should have 2 webseeds on the first video', async function () {
 562       this.timeout(80000)
 563
 564       await waitJobs(servers)
 565       await waitUntilLog(servers[0], 'Duplicated ', 5)
 566       await waitJobs(servers)
 567
 568       await check2Webseeds()
 569       await check1PlaylistRedundancies()
 570       await checkStatsWith1Redundancy('manual')
 571     })
 572
 573     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
 574       this.timeout(80000)
 575
 576       const res = await listVideoRedundancies({
 577         url: servers[0].url,
 578         accessToken: servers[0].accessToken,
 579         target: 'remote-videos'
 580       })
 581
 582       const videos = res.body.data as VideoRedundancy[]
 583       expect(videos).to.have.lengthOf(1)
 584
 585       const video = videos[0]
 586       for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
 587         await removeVideoRedundancy({
 588           url: servers[0].url,
 589           accessToken: servers[0].accessToken,
 590           redundancyId: r.id
 591         })
 592       }
 593
 594       await waitJobs(servers)
 595       await wait(5000)
 596
 597       await check1WebSeed()
 598       await check0PlaylistRedundancies()
 599
 600       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 601     })
 602
 603     after(async function () {
 604       await cleanupTests(servers)
 605     })
 606   })
 607
 608   describe('Test expiration', function () {
 609     const strategy = 'recently-added'
 610
 611     async function checkContains (servers: ServerInfo[], str: string) {
 612       for (const server of servers) {
 613         const res = await getVideo(server.url, video1Server2UUID)
 614         const video: VideoDetails = res.body
 615
 616         for (const f of video.files) {
 617           expect(f.magnetUri).to.contain(str)
 618         }
 619       }
 620     }
 621
 622     async function checkNotContains (servers: ServerInfo[], str: string) {
 623       for (const server of servers) {
 624         const res = await getVideo(server.url, video1Server2UUID)
 625         const video: VideoDetails = res.body
 626
 627         for (const f of video.files) {
 628           expect(f.magnetUri).to.not.contain(str)
 629         }
 630       }
 631     }
 632
 633     before(async function () {
 634       this.timeout(120000)
 635
 636       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 637
 638       await enableRedundancyOnServer1()
 639     })
 640
 641     it('Should still have 2 webseeds after 10 seconds', async function () {
 642       this.timeout(80000)
 643
 644       await wait(10000)
 645
 646       try {
 647         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 648       } catch {
 649         // Maybe a server deleted a redundancy in the scheduler
 650         await wait(2000)
 651
 652         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 653       }
 654     })
 655
 656     it('Should stop server 1 and expire video redundancy', async function () {
 657       this.timeout(80000)
 658
 659       killallServers([ servers[0] ])
 660
 661       await wait(15000)
 662
 663       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 664     })
 665
 666     after(async function () {
 667       await cleanupTests(servers)
 668     })
 669   })
 670
 671   describe('Test file replacement', function () {
 672     let video2Server2UUID: string
 673     const strategy = 'recently-added'
 674
 675     before(async function () {
 676       this.timeout(120000)
 677
 678       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 679
 680       await enableRedundancyOnServer1()
 681
 682       await waitJobs(servers)
 683       await waitUntilLog(servers[0], 'Duplicated ', 5)
 684       await waitJobs(servers)
 685
 686       await check2Webseeds(video1Server2UUID)
 687       await check1PlaylistRedundancies(video1Server2UUID)
 688       await checkStatsWith1Redundancy(strategy)
 689
 690       const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE })
 691       video2Server2UUID = res.body.video.uuid
 692
 693       // Wait transcoding before federation
 694       await waitJobs(servers)
 695
 696       await updateVideo(servers[1].url, servers[1].accessToken, video2Server2UUID, { privacy: VideoPrivacy.PUBLIC })
 697     })
 698
 699     it('Should cache video 2 webseeds on the first video', async function () {
 700       this.timeout(120000)
 701
 702       await waitJobs(servers)
 703
 704       let checked = false
 705
 706       while (checked === false) {
 707         await wait(1000)
 708
 709         try {
 710           await check1WebSeed(video1Server2UUID)
 711           await check0PlaylistRedundancies(video1Server2UUID)
 712
 713           await check2Webseeds(video2Server2UUID)
 714           await check1PlaylistRedundancies(video2Server2UUID)
 715
 716           checked = true
 717         } catch {
 718           checked = false
 719         }
 720       }
 721     })
 722
 723     it('Should disable strategy and remove redundancies', async function () {
 724       this.timeout(80000)
 725
 726       await waitJobs(servers)
 727
 728       killallServers([ servers[0] ])
 729       await reRunServer(servers[0], {
 730         redundancy: {
 731           videos: {
 732             check_interval: '1 second',
 733             strategies: []
 734           }
 735         }
 736       })
 737
 738       await waitJobs(servers)
 739
 740       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
 741     })
 742
 743     after(async function () {
 744       await cleanupTests(servers)
 745     })
 746   })
 747 })