server/tests/api/redundancy/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   doubleFollow,
   8   flushAndRunMultipleServers,
   9   getFollowingListPaginationAndSort,
  10   getVideo,
  11   immutableAssign,
  12   killallServers, makeGetRequest,
  13   root,
  14   ServerInfo,
  15   setAccessTokensToServers, unfollow,
  16   uploadVideo,
  17   viewVideo,
  18   wait,
  19   waitUntilLog,
  20   checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer, checkSegmentHash
  21 } from '../../../../shared/utils'
  22 import { waitJobs } from '../../../../shared/utils/server/jobs'
  23
  24 import * as magnetUtil from 'magnet-uri'
  25 import { updateRedundancy } from '../../../../shared/utils/server/redundancy'
  26 import { ActorFollow } from '../../../../shared/models/actors'
  27 import { readdir } from 'fs-extra'
  28 import { join } from 'path'
  29 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  30 import { getStats } from '../../../../shared/utils/server/stats'
  31 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  32
  33 const expect = chai.expect
  34
  35 let servers: ServerInfo[] = []
  36 let video1Server2UUID: string
  37
  38 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  39   const parsed = magnetUtil.decode(file.magnetUri)
  40
  41   for (const ws of baseWebseeds) {
  42     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  43     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  44   }
  45
  46   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  47 }
  48
  49 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  50   const config = {
  51     transcoding: {
  52       hls: {
  53         enabled: true
  54       }
  55     },
  56     redundancy: {
  57       videos: {
  58         check_interval: '5 seconds',
  59         strategies: [
  60           immutableAssign({
  61             min_lifetime: '1 hour',
  62             strategy: strategy,
  63             size: '200KB'
  64           }, additionalParams)
  65         ]
  66       }
  67     }
  68   }
  69   servers = await flushAndRunMultipleServers(3, config)
  70
  71   // Get the access tokens
  72   await setAccessTokensToServers(servers)
  73
  74   {
  75     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  76     video1Server2UUID = res.body.video.uuid
  77
  78     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  79   }
  80
  81   await waitJobs(servers)
  82
  83   // Server 1 and server 2 follow each other
  84   await doubleFollow(servers[ 0 ], servers[ 1 ])
  85   // Server 1 and server 3 follow each other
  86   await doubleFollow(servers[ 0 ], servers[ 2 ])
  87   // Server 2 and server 3 follow each other
  88   await doubleFollow(servers[ 1 ], servers[ 2 ])
  89
  90   await waitJobs(servers)
  91 }
  92
  93 async function check1WebSeed (videoUUID?: string) {
  94   if (!videoUUID) videoUUID = video1Server2UUID
  95
  96   const webseeds = [
  97     'http://localhost:9002/static/webseed/' + videoUUID
  98   ]
  99
 100   for (const server of servers) {
 101     // With token to avoid issues with video follow constraints
 102     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 103
 104     const video: VideoDetails = res.body
 105     for (const f of video.files) {
 106       checkMagnetWebseeds(f, webseeds, server)
 107     }
 108   }
 109 }
 110
 111 async function check2Webseeds (videoUUID?: string) {
 112   if (!videoUUID) videoUUID = video1Server2UUID
 113
 114   const webseeds = [
 115     'http://localhost:9001/static/redundancy/' + videoUUID,
 116     'http://localhost:9002/static/webseed/' + videoUUID
 117   ]
 118
 119   for (const server of servers) {
 120     const res = await getVideo(server.url, videoUUID)
 121
 122     const video: VideoDetails = res.body
 123
 124     for (const file of video.files) {
 125       checkMagnetWebseeds(file, webseeds, server)
 126
 127       await makeGetRequest({
 128         url: servers[0].url,
 129         statusCodeExpected: 200,
 130         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 131         contentType: null
 132       })
 133       await makeGetRequest({
 134         url: servers[1].url,
 135         statusCodeExpected: 200,
 136         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 137         contentType: null
 138       })
 139     }
 140   }
 141
 142   for (const directory of [ 'test1/redundancy', 'test2/videos' ]) {
 143     const files = await readdir(join(root(), directory))
 144     expect(files).to.have.length.at.least(4)
 145
 146     for (const resolution of [ 240, 360, 480, 720 ]) {
 147       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 148     }
 149   }
 150 }
 151
 152 async function check0PlaylistRedundancies (videoUUID?: string) {
 153   if (!videoUUID) videoUUID = video1Server2UUID
 154
 155   for (const server of servers) {
 156     // With token to avoid issues with video follow constraints
 157     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 158     const video: VideoDetails = res.body
 159
 160     expect(video.streamingPlaylists).to.be.an('array')
 161     expect(video.streamingPlaylists).to.have.lengthOf(1)
 162     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 163   }
 164 }
 165
 166 async function check1PlaylistRedundancies (videoUUID?: string) {
 167   if (!videoUUID) videoUUID = video1Server2UUID
 168
 169   for (const server of servers) {
 170     const res = await getVideo(server.url, videoUUID)
 171     const video: VideoDetails = res.body
 172
 173     expect(video.streamingPlaylists).to.have.lengthOf(1)
 174     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 175
 176     const redundancy = video.streamingPlaylists[0].redundancies[0]
 177
 178     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 179   }
 180
 181   const baseUrlPlaylist = servers[1].url + '/static/playlists/hls'
 182   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 183
 184   const res = await getVideo(servers[0].url, videoUUID)
 185   const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
 186
 187   for (const resolution of [ 240, 360, 480, 720 ]) {
 188     await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
 189   }
 190
 191   for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) {
 192     const files = await readdir(join(root(), directory, videoUUID))
 193     expect(files).to.have.length.at.least(4)
 194
 195     for (const resolution of [ 240, 360, 480, 720 ]) {
 196       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 197
 198       expect(files.find(f => f === filename)).to.not.be.undefined
 199     }
 200   }
 201 }
 202
 203 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 204   const res = await getStats(servers[0].url)
 205   const data: ServerStats = res.body
 206
 207   expect(data.videosRedundancy).to.have.lengthOf(1)
 208   const stat = data.videosRedundancy[0]
 209
 210   expect(stat.strategy).to.equal(strategy)
 211   expect(stat.totalSize).to.equal(204800)
 212   expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
 213   expect(stat.totalVideoFiles).to.equal(4)
 214   expect(stat.totalVideos).to.equal(1)
 215 }
 216
 217 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 218   const res = await getStats(servers[0].url)
 219   const data: ServerStats = res.body
 220
 221   expect(data.videosRedundancy).to.have.lengthOf(1)
 222
 223   const stat = data.videosRedundancy[0]
 224   expect(stat.strategy).to.equal(strategy)
 225   expect(stat.totalSize).to.equal(204800)
 226   expect(stat.totalUsed).to.equal(0)
 227   expect(stat.totalVideoFiles).to.equal(0)
 228   expect(stat.totalVideos).to.equal(0)
 229 }
 230
 231 async function enableRedundancyOnServer1 () {
 232   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 233
 234   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 235   const follows: ActorFollow[] = res.body.data
 236   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 237   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 238
 239   expect(server3).to.not.be.undefined
 240   expect(server3.following.hostRedundancyAllowed).to.be.false
 241
 242   expect(server2).to.not.be.undefined
 243   expect(server2.following.hostRedundancyAllowed).to.be.true
 244 }
 245
 246 async function disableRedundancyOnServer1 () {
 247   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
 248
 249   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 250   const follows: ActorFollow[] = res.body.data
 251   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 252   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 253
 254   expect(server3).to.not.be.undefined
 255   expect(server3.following.hostRedundancyAllowed).to.be.false
 256
 257   expect(server2).to.not.be.undefined
 258   expect(server2.following.hostRedundancyAllowed).to.be.false
 259 }
 260
 261 async function cleanServers () {
 262   killallServers(servers)
 263 }
 264
 265 describe('Test videos redundancy', function () {
 266
 267   describe('With most-views strategy', function () {
 268     const strategy = 'most-views'
 269
 270     before(function () {
 271       this.timeout(120000)
 272
 273       return runServers(strategy)
 274     })
 275
 276     it('Should have 1 webseed on the first video', async function () {
 277       await check1WebSeed()
 278       await check0PlaylistRedundancies()
 279       await checkStatsWith1Webseed(strategy)
 280     })
 281
 282     it('Should enable redundancy on server 1', function () {
 283       return enableRedundancyOnServer1()
 284     })
 285
 286     it('Should have 2 webseeds on the first video', async function () {
 287       this.timeout(80000)
 288
 289       await waitJobs(servers)
 290       await waitUntilLog(servers[0], 'Duplicated ', 5)
 291       await waitJobs(servers)
 292
 293       await check2Webseeds()
 294       await check1PlaylistRedundancies()
 295       await checkStatsWith2Webseed(strategy)
 296     })
 297
 298     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 299       this.timeout(80000)
 300
 301       await disableRedundancyOnServer1()
 302
 303       await waitJobs(servers)
 304       await wait(5000)
 305
 306       await check1WebSeed()
 307       await check0PlaylistRedundancies()
 308
 309       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
 310     })
 311
 312     after(function () {
 313       return cleanServers()
 314     })
 315   })
 316
 317   describe('With trending strategy', function () {
 318     const strategy = 'trending'
 319
 320     before(function () {
 321       this.timeout(120000)
 322
 323       return runServers(strategy)
 324     })
 325
 326     it('Should have 1 webseed on the first video', async function () {
 327       await check1WebSeed()
 328       await check0PlaylistRedundancies()
 329       await checkStatsWith1Webseed(strategy)
 330     })
 331
 332     it('Should enable redundancy on server 1', function () {
 333       return enableRedundancyOnServer1()
 334     })
 335
 336     it('Should have 2 webseeds on the first video', async function () {
 337       this.timeout(80000)
 338
 339       await waitJobs(servers)
 340       await waitUntilLog(servers[0], 'Duplicated ', 5)
 341       await waitJobs(servers)
 342
 343       await check2Webseeds()
 344       await check1PlaylistRedundancies()
 345       await checkStatsWith2Webseed(strategy)
 346     })
 347
 348     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 349       this.timeout(80000)
 350
 351       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 352
 353       await waitJobs(servers)
 354       await wait(5000)
 355
 356       await check1WebSeed()
 357       await check0PlaylistRedundancies()
 358
 359       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 360     })
 361
 362     after(function () {
 363       return cleanServers()
 364     })
 365   })
 366
 367   describe('With recently added strategy', function () {
 368     const strategy = 'recently-added'
 369
 370     before(function () {
 371       this.timeout(120000)
 372
 373       return runServers(strategy, { min_views: 3 })
 374     })
 375
 376     it('Should have 1 webseed on the first video', async function () {
 377       await check1WebSeed()
 378       await check0PlaylistRedundancies()
 379       await checkStatsWith1Webseed(strategy)
 380     })
 381
 382     it('Should enable redundancy on server 1', function () {
 383       return enableRedundancyOnServer1()
 384     })
 385
 386     it('Should still have 1 webseed on the first video', async function () {
 387       this.timeout(80000)
 388
 389       await waitJobs(servers)
 390       await wait(15000)
 391       await waitJobs(servers)
 392
 393       await check1WebSeed()
 394       await check0PlaylistRedundancies()
 395       await checkStatsWith1Webseed(strategy)
 396     })
 397
 398     it('Should view 2 times the first video to have > min_views config', async function () {
 399       this.timeout(80000)
 400
 401       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 402       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 403
 404       await wait(10000)
 405       await waitJobs(servers)
 406     })
 407
 408     it('Should have 2 webseeds on the first video', async function () {
 409       this.timeout(80000)
 410
 411       await waitJobs(servers)
 412       await waitUntilLog(servers[0], 'Duplicated ', 5)
 413       await waitJobs(servers)
 414
 415       await check2Webseeds()
 416       await check1PlaylistRedundancies()
 417       await checkStatsWith2Webseed(strategy)
 418     })
 419
 420     it('Should remove the video and the redundancy files', async function () {
 421       this.timeout(20000)
 422
 423       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 424
 425       await waitJobs(servers)
 426
 427       for (const server of servers) {
 428         await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber)
 429       }
 430     })
 431
 432     after(function () {
 433       return cleanServers()
 434     })
 435   })
 436
 437   describe('Test expiration', function () {
 438     const strategy = 'recently-added'
 439
 440     async function checkContains (servers: ServerInfo[], str: string) {
 441       for (const server of servers) {
 442         const res = await getVideo(server.url, video1Server2UUID)
 443         const video: VideoDetails = res.body
 444
 445         for (const f of video.files) {
 446           expect(f.magnetUri).to.contain(str)
 447         }
 448       }
 449     }
 450
 451     async function checkNotContains (servers: ServerInfo[], str: string) {
 452       for (const server of servers) {
 453         const res = await getVideo(server.url, video1Server2UUID)
 454         const video: VideoDetails = res.body
 455
 456         for (const f of video.files) {
 457           expect(f.magnetUri).to.not.contain(str)
 458         }
 459       }
 460     }
 461
 462     before(async function () {
 463       this.timeout(120000)
 464
 465       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 466
 467       await enableRedundancyOnServer1()
 468     })
 469
 470     it('Should still have 2 webseeds after 10 seconds', async function () {
 471       this.timeout(80000)
 472
 473       await wait(10000)
 474
 475       try {
 476         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 477       } catch {
 478         // Maybe a server deleted a redundancy in the scheduler
 479         await wait(2000)
 480
 481         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 482       }
 483     })
 484
 485     it('Should stop server 1 and expire video redundancy', async function () {
 486       this.timeout(80000)
 487
 488       killallServers([ servers[0] ])
 489
 490       await wait(15000)
 491
 492       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 493     })
 494
 495     after(function () {
 496       return killallServers([ servers[1], servers[2] ])
 497     })
 498   })
 499
 500   describe('Test file replacement', function () {
 501     let video2Server2UUID: string
 502     const strategy = 'recently-added'
 503
 504     before(async function () {
 505       this.timeout(120000)
 506
 507       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 508
 509       await enableRedundancyOnServer1()
 510
 511       await waitJobs(servers)
 512       await waitUntilLog(servers[0], 'Duplicated ', 5)
 513       await waitJobs(servers)
 514
 515       await check2Webseeds()
 516       await check1PlaylistRedundancies()
 517       await checkStatsWith2Webseed(strategy)
 518
 519       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 520       video2Server2UUID = res.body.video.uuid
 521     })
 522
 523     it('Should cache video 2 webseeds on the first video', async function () {
 524       this.timeout(120000)
 525
 526       await waitJobs(servers)
 527
 528       let checked = false
 529
 530       while (checked === false) {
 531         await wait(1000)
 532
 533         try {
 534           await check1WebSeed(video1Server2UUID)
 535           await check0PlaylistRedundancies(video1Server2UUID)
 536           await check2Webseeds(video2Server2UUID)
 537           await check1PlaylistRedundancies(video2Server2UUID)
 538
 539           checked = true
 540         } catch {
 541           checked = false
 542         }
 543       }
 544     })
 545
 546     it('Should disable strategy and remove redundancies', async function () {
 547       this.timeout(80000)
 548
 549       await waitJobs(servers)
 550
 551       killallServers([ servers[ 0 ] ])
 552       await reRunServer(servers[ 0 ], {
 553         redundancy: {
 554           videos: {
 555             check_interval: '1 second',
 556             strategies: []
 557           }
 558         }
 559       })
 560
 561       await waitJobs(servers)
 562
 563       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
 564     })
 565
 566     after(function () {
 567       return cleanServers()
 568     })
 569   })
 570 })