server/tests/api/redundancy/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   checkSegmentHash,
   8   checkVideoFilesWereRemoved,
   9   doubleFollow,
  10   flushAndRunMultipleServers,
  11   getFollowingListPaginationAndSort,
  12   getVideo,
  13   getVideoWithToken,
  14   immutableAssign,
  15   killallServers,
  16   makeGetRequest,
  17   removeVideo,
  18   reRunServer,
  19   root,
  20   ServerInfo,
  21   setAccessTokensToServers,
  22   unfollow,
  23   uploadVideo,
  24   viewVideo,
  25   wait,
  26   waitUntilLog
  27 } from '../../../../shared/utils'
  28 import { waitJobs } from '../../../../shared/utils/server/jobs'
  29
  30 import * as magnetUtil from 'magnet-uri'
  31 import { updateRedundancy } from '../../../../shared/utils/server/redundancy'
  32 import { ActorFollow } from '../../../../shared/models/actors'
  33 import { readdir } from 'fs-extra'
  34 import { join } from 'path'
  35 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  36 import { getStats } from '../../../../shared/utils/server/stats'
  37 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  38
  39 const expect = chai.expect
  40
  41 let servers: ServerInfo[] = []
  42 let video1Server2UUID: string
  43
  44 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  45   const parsed = magnetUtil.decode(file.magnetUri)
  46
  47   for (const ws of baseWebseeds) {
  48     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  49     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  50   }
  51
  52   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  53 }
  54
  55 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  56   const config = {
  57     transcoding: {
  58       hls: {
  59         enabled: true
  60       }
  61     },
  62     redundancy: {
  63       videos: {
  64         check_interval: '5 seconds',
  65         strategies: [
  66           immutableAssign({
  67             min_lifetime: '1 hour',
  68             strategy: strategy,
  69             size: '200KB'
  70           }, additionalParams)
  71         ]
  72       }
  73     }
  74   }
  75   servers = await flushAndRunMultipleServers(3, config)
  76
  77   // Get the access tokens
  78   await setAccessTokensToServers(servers)
  79
  80   {
  81     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  82     video1Server2UUID = res.body.video.uuid
  83
  84     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  85   }
  86
  87   await waitJobs(servers)
  88
  89   // Server 1 and server 2 follow each other
  90   await doubleFollow(servers[ 0 ], servers[ 1 ])
  91   // Server 1 and server 3 follow each other
  92   await doubleFollow(servers[ 0 ], servers[ 2 ])
  93   // Server 2 and server 3 follow each other
  94   await doubleFollow(servers[ 1 ], servers[ 2 ])
  95
  96   await waitJobs(servers)
  97 }
  98
  99 async function check1WebSeed (videoUUID?: string) {
 100   if (!videoUUID) videoUUID = video1Server2UUID
 101
 102   const webseeds = [
 103     'http://localhost:9002/static/webseed/' + videoUUID
 104   ]
 105
 106   for (const server of servers) {
 107     // With token to avoid issues with video follow constraints
 108     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 109
 110     const video: VideoDetails = res.body
 111     for (const f of video.files) {
 112       checkMagnetWebseeds(f, webseeds, server)
 113     }
 114   }
 115 }
 116
 117 async function check2Webseeds (videoUUID?: string) {
 118   if (!videoUUID) videoUUID = video1Server2UUID
 119
 120   const webseeds = [
 121     'http://localhost:9001/static/redundancy/' + videoUUID,
 122     'http://localhost:9002/static/webseed/' + videoUUID
 123   ]
 124
 125   for (const server of servers) {
 126     const res = await getVideo(server.url, videoUUID)
 127
 128     const video: VideoDetails = res.body
 129
 130     for (const file of video.files) {
 131       checkMagnetWebseeds(file, webseeds, server)
 132
 133       await makeGetRequest({
 134         url: servers[0].url,
 135         statusCodeExpected: 200,
 136         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 137         contentType: null
 138       })
 139       await makeGetRequest({
 140         url: servers[1].url,
 141         statusCodeExpected: 200,
 142         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 143         contentType: null
 144       })
 145     }
 146   }
 147
 148   for (const directory of [ 'test1/redundancy', 'test2/videos' ]) {
 149     const files = await readdir(join(root(), directory))
 150     expect(files).to.have.length.at.least(4)
 151
 152     for (const resolution of [ 240, 360, 480, 720 ]) {
 153       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 154     }
 155   }
 156 }
 157
 158 async function check0PlaylistRedundancies (videoUUID?: string) {
 159   if (!videoUUID) videoUUID = video1Server2UUID
 160
 161   for (const server of servers) {
 162     // With token to avoid issues with video follow constraints
 163     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 164     const video: VideoDetails = res.body
 165
 166     expect(video.streamingPlaylists).to.be.an('array')
 167     expect(video.streamingPlaylists).to.have.lengthOf(1)
 168     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 169   }
 170 }
 171
 172 async function check1PlaylistRedundancies (videoUUID?: string) {
 173   if (!videoUUID) videoUUID = video1Server2UUID
 174
 175   for (const server of servers) {
 176     const res = await getVideo(server.url, videoUUID)
 177     const video: VideoDetails = res.body
 178
 179     expect(video.streamingPlaylists).to.have.lengthOf(1)
 180     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 181
 182     const redundancy = video.streamingPlaylists[0].redundancies[0]
 183
 184     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 185   }
 186
 187   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 188   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 189
 190   const res = await getVideo(servers[0].url, videoUUID)
 191   const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
 192
 193   for (const resolution of [ 240, 360, 480, 720 ]) {
 194     await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
 195   }
 196
 197   for (const directory of [ 'test1/redundancy/hls', 'test2/streaming-playlists/hls' ]) {
 198     const files = await readdir(join(root(), directory, videoUUID))
 199     expect(files).to.have.length.at.least(4)
 200
 201     for (const resolution of [ 240, 360, 480, 720 ]) {
 202       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 203
 204       expect(files.find(f => f === filename)).to.not.be.undefined
 205     }
 206   }
 207 }
 208
 209 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 210   const res = await getStats(servers[0].url)
 211   const data: ServerStats = res.body
 212
 213   expect(data.videosRedundancy).to.have.lengthOf(1)
 214   const stat = data.videosRedundancy[0]
 215
 216   expect(stat.strategy).to.equal(strategy)
 217   expect(stat.totalSize).to.equal(204800)
 218   expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
 219   expect(stat.totalVideoFiles).to.equal(4)
 220   expect(stat.totalVideos).to.equal(1)
 221 }
 222
 223 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 224   const res = await getStats(servers[0].url)
 225   const data: ServerStats = res.body
 226
 227   expect(data.videosRedundancy).to.have.lengthOf(1)
 228
 229   const stat = data.videosRedundancy[0]
 230   expect(stat.strategy).to.equal(strategy)
 231   expect(stat.totalSize).to.equal(204800)
 232   expect(stat.totalUsed).to.equal(0)
 233   expect(stat.totalVideoFiles).to.equal(0)
 234   expect(stat.totalVideos).to.equal(0)
 235 }
 236
 237 async function enableRedundancyOnServer1 () {
 238   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 239
 240   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 241   const follows: ActorFollow[] = res.body.data
 242   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 243   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 244
 245   expect(server3).to.not.be.undefined
 246   expect(server3.following.hostRedundancyAllowed).to.be.false
 247
 248   expect(server2).to.not.be.undefined
 249   expect(server2.following.hostRedundancyAllowed).to.be.true
 250 }
 251
 252 async function disableRedundancyOnServer1 () {
 253   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
 254
 255   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 256   const follows: ActorFollow[] = res.body.data
 257   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 258   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 259
 260   expect(server3).to.not.be.undefined
 261   expect(server3.following.hostRedundancyAllowed).to.be.false
 262
 263   expect(server2).to.not.be.undefined
 264   expect(server2.following.hostRedundancyAllowed).to.be.false
 265 }
 266
 267 async function cleanServers () {
 268   killallServers(servers)
 269 }
 270
 271 describe('Test videos redundancy', function () {
 272
 273   describe('With most-views strategy', function () {
 274     const strategy = 'most-views'
 275
 276     before(function () {
 277       this.timeout(120000)
 278
 279       return runServers(strategy)
 280     })
 281
 282     it('Should have 1 webseed on the first video', async function () {
 283       await check1WebSeed()
 284       await check0PlaylistRedundancies()
 285       await checkStatsWith1Webseed(strategy)
 286     })
 287
 288     it('Should enable redundancy on server 1', function () {
 289       return enableRedundancyOnServer1()
 290     })
 291
 292     it('Should have 2 webseeds on the first video', async function () {
 293       this.timeout(80000)
 294
 295       await waitJobs(servers)
 296       await waitUntilLog(servers[0], 'Duplicated ', 5)
 297       await waitJobs(servers)
 298
 299       await check2Webseeds()
 300       await check1PlaylistRedundancies()
 301       await checkStatsWith2Webseed(strategy)
 302     })
 303
 304     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 305       this.timeout(80000)
 306
 307       await disableRedundancyOnServer1()
 308
 309       await waitJobs(servers)
 310       await wait(5000)
 311
 312       await check1WebSeed()
 313       await check0PlaylistRedundancies()
 314
 315       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
 316     })
 317
 318     after(function () {
 319       return cleanServers()
 320     })
 321   })
 322
 323   describe('With trending strategy', function () {
 324     const strategy = 'trending'
 325
 326     before(function () {
 327       this.timeout(120000)
 328
 329       return runServers(strategy)
 330     })
 331
 332     it('Should have 1 webseed on the first video', async function () {
 333       await check1WebSeed()
 334       await check0PlaylistRedundancies()
 335       await checkStatsWith1Webseed(strategy)
 336     })
 337
 338     it('Should enable redundancy on server 1', function () {
 339       return enableRedundancyOnServer1()
 340     })
 341
 342     it('Should have 2 webseeds on the first video', async function () {
 343       this.timeout(80000)
 344
 345       await waitJobs(servers)
 346       await waitUntilLog(servers[0], 'Duplicated ', 5)
 347       await waitJobs(servers)
 348
 349       await check2Webseeds()
 350       await check1PlaylistRedundancies()
 351       await checkStatsWith2Webseed(strategy)
 352     })
 353
 354     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 355       this.timeout(80000)
 356
 357       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 358
 359       await waitJobs(servers)
 360       await wait(5000)
 361
 362       await check1WebSeed()
 363       await check0PlaylistRedundancies()
 364
 365       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 366     })
 367
 368     after(function () {
 369       return cleanServers()
 370     })
 371   })
 372
 373   describe('With recently added strategy', function () {
 374     const strategy = 'recently-added'
 375
 376     before(function () {
 377       this.timeout(120000)
 378
 379       return runServers(strategy, { min_views: 3 })
 380     })
 381
 382     it('Should have 1 webseed on the first video', async function () {
 383       await check1WebSeed()
 384       await check0PlaylistRedundancies()
 385       await checkStatsWith1Webseed(strategy)
 386     })
 387
 388     it('Should enable redundancy on server 1', function () {
 389       return enableRedundancyOnServer1()
 390     })
 391
 392     it('Should still have 1 webseed on the first video', async function () {
 393       this.timeout(80000)
 394
 395       await waitJobs(servers)
 396       await wait(15000)
 397       await waitJobs(servers)
 398
 399       await check1WebSeed()
 400       await check0PlaylistRedundancies()
 401       await checkStatsWith1Webseed(strategy)
 402     })
 403
 404     it('Should view 2 times the first video to have > min_views config', async function () {
 405       this.timeout(80000)
 406
 407       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 408       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 409
 410       await wait(10000)
 411       await waitJobs(servers)
 412     })
 413
 414     it('Should have 2 webseeds on the first video', async function () {
 415       this.timeout(80000)
 416
 417       await waitJobs(servers)
 418       await waitUntilLog(servers[0], 'Duplicated ', 5)
 419       await waitJobs(servers)
 420
 421       await check2Webseeds()
 422       await check1PlaylistRedundancies()
 423       await checkStatsWith2Webseed(strategy)
 424     })
 425
 426     it('Should remove the video and the redundancy files', async function () {
 427       this.timeout(20000)
 428
 429       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 430
 431       await waitJobs(servers)
 432
 433       for (const server of servers) {
 434         await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber)
 435       }
 436     })
 437
 438     after(function () {
 439       return cleanServers()
 440     })
 441   })
 442
 443   describe('Test expiration', function () {
 444     const strategy = 'recently-added'
 445
 446     async function checkContains (servers: ServerInfo[], str: string) {
 447       for (const server of servers) {
 448         const res = await getVideo(server.url, video1Server2UUID)
 449         const video: VideoDetails = res.body
 450
 451         for (const f of video.files) {
 452           expect(f.magnetUri).to.contain(str)
 453         }
 454       }
 455     }
 456
 457     async function checkNotContains (servers: ServerInfo[], str: string) {
 458       for (const server of servers) {
 459         const res = await getVideo(server.url, video1Server2UUID)
 460         const video: VideoDetails = res.body
 461
 462         for (const f of video.files) {
 463           expect(f.magnetUri).to.not.contain(str)
 464         }
 465       }
 466     }
 467
 468     before(async function () {
 469       this.timeout(120000)
 470
 471       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 472
 473       await enableRedundancyOnServer1()
 474     })
 475
 476     it('Should still have 2 webseeds after 10 seconds', async function () {
 477       this.timeout(80000)
 478
 479       await wait(10000)
 480
 481       try {
 482         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 483       } catch {
 484         // Maybe a server deleted a redundancy in the scheduler
 485         await wait(2000)
 486
 487         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 488       }
 489     })
 490
 491     it('Should stop server 1 and expire video redundancy', async function () {
 492       this.timeout(80000)
 493
 494       killallServers([ servers[0] ])
 495
 496       await wait(15000)
 497
 498       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 499     })
 500
 501     after(function () {
 502       return killallServers([ servers[1], servers[2] ])
 503     })
 504   })
 505
 506   describe('Test file replacement', function () {
 507     let video2Server2UUID: string
 508     const strategy = 'recently-added'
 509
 510     before(async function () {
 511       this.timeout(120000)
 512
 513       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 514
 515       await enableRedundancyOnServer1()
 516
 517       await waitJobs(servers)
 518       await waitUntilLog(servers[0], 'Duplicated ', 5)
 519       await waitJobs(servers)
 520
 521       await check2Webseeds()
 522       await check1PlaylistRedundancies()
 523       await checkStatsWith2Webseed(strategy)
 524
 525       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 526       video2Server2UUID = res.body.video.uuid
 527     })
 528
 529     it('Should cache video 2 webseeds on the first video', async function () {
 530       this.timeout(120000)
 531
 532       await waitJobs(servers)
 533
 534       let checked = false
 535
 536       while (checked === false) {
 537         await wait(1000)
 538
 539         try {
 540           await check1WebSeed(video1Server2UUID)
 541           await check0PlaylistRedundancies(video1Server2UUID)
 542           await check2Webseeds(video2Server2UUID)
 543           await check1PlaylistRedundancies(video2Server2UUID)
 544
 545           checked = true
 546         } catch {
 547           checked = false
 548         }
 549       }
 550     })
 551
 552     it('Should disable strategy and remove redundancies', async function () {
 553       this.timeout(80000)
 554
 555       await waitJobs(servers)
 556
 557       killallServers([ servers[ 0 ] ])
 558       await reRunServer(servers[ 0 ], {
 559         redundancy: {
 560           videos: {
 561             check_interval: '1 second',
 562             strategies: []
 563           }
 564         }
 565       })
 566
 567       await waitJobs(servers)
 568
 569       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
 570     })
 571
 572     after(function () {
 573       return cleanServers()
 574     })
 575   })
 576 })