server/tests/api/redundancy/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   checkSegmentHash,
   8   checkVideoFilesWereRemoved, cleanupTests,
   9   doubleFollow,
  10   flushAndRunMultipleServers,
  11   getFollowingListPaginationAndSort,
  12   getVideo,
  13   getVideoWithToken,
  14   immutableAssign,
  15   killallServers,
  16   makeGetRequest,
  17   removeVideo,
  18   reRunServer,
  19   root,
  20   ServerInfo,
  21   setAccessTokensToServers,
  22   unfollow,
  23   uploadVideo,
  24   viewVideo,
  25   wait,
  26   waitUntilLog
  27 } from '../../../../shared/extra-utils'
  28 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
  29
  30 import * as magnetUtil from 'magnet-uri'
  31 import { updateRedundancy } from '../../../../shared/extra-utils/server/redundancy'
  32 import { ActorFollow } from '../../../../shared/models/actors'
  33 import { readdir } from 'fs-extra'
  34 import { join } from 'path'
  35 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  36 import { getStats } from '../../../../shared/extra-utils/server/stats'
  37 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  38
  39 const expect = chai.expect
  40
  41 let servers: ServerInfo[] = []
  42 let video1Server2UUID: string
  43
  44 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  45   const parsed = magnetUtil.decode(file.magnetUri)
  46
  47   for (const ws of baseWebseeds) {
  48     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  49     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  50   }
  51
  52   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  53 }
  54
  55 async function flushAndRunServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  56   const config = {
  57     transcoding: {
  58       hls: {
  59         enabled: true
  60       }
  61     },
  62     redundancy: {
  63       videos: {
  64         check_interval: '5 seconds',
  65         strategies: [
  66           immutableAssign({
  67             min_lifetime: '1 hour',
  68             strategy: strategy,
  69             size: '400KB'
  70           }, additionalParams)
  71         ]
  72       }
  73     }
  74   }
  75   servers = await flushAndRunMultipleServers(3, config)
  76
  77   // Get the access tokens
  78   await setAccessTokensToServers(servers)
  79
  80   {
  81     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  82     video1Server2UUID = res.body.video.uuid
  83
  84     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  85   }
  86
  87   await waitJobs(servers)
  88
  89   // Server 1 and server 2 follow each other
  90   await doubleFollow(servers[ 0 ], servers[ 1 ])
  91   // Server 1 and server 3 follow each other
  92   await doubleFollow(servers[ 0 ], servers[ 2 ])
  93   // Server 2 and server 3 follow each other
  94   await doubleFollow(servers[ 1 ], servers[ 2 ])
  95
  96   await waitJobs(servers)
  97 }
  98
  99 async function check1WebSeed (videoUUID?: string) {
 100   if (!videoUUID) videoUUID = video1Server2UUID
 101
 102   const webseeds = [
 103     `http://localhost:${servers[ 1 ].port}/static/webseed/${videoUUID}`
 104   ]
 105
 106   for (const server of servers) {
 107     // With token to avoid issues with video follow constraints
 108     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 109
 110     const video: VideoDetails = res.body
 111     for (const f of video.files) {
 112       checkMagnetWebseeds(f, webseeds, server)
 113     }
 114   }
 115 }
 116
 117 async function check2Webseeds (videoUUID?: string) {
 118   if (!videoUUID) videoUUID = video1Server2UUID
 119
 120   const webseeds = [
 121     `http://localhost:${servers[ 0 ].port}/static/redundancy/${videoUUID}`,
 122     `http://localhost:${servers[ 1 ].port}/static/webseed/${videoUUID}`
 123   ]
 124
 125   for (const server of servers) {
 126     const res = await getVideo(server.url, videoUUID)
 127
 128     const video: VideoDetails = res.body
 129
 130     for (const file of video.files) {
 131       checkMagnetWebseeds(file, webseeds, server)
 132
 133       await makeGetRequest({
 134         url: servers[0].url,
 135         statusCodeExpected: 200,
 136         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 137         contentType: null
 138       })
 139       await makeGetRequest({
 140         url: servers[1].url,
 141         statusCodeExpected: 200,
 142         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 143         contentType: null
 144       })
 145     }
 146   }
 147
 148   const directories = [
 149     'test' + servers[0].internalServerNumber + '/redundancy',
 150     'test' + servers[1].internalServerNumber + '/videos'
 151   ]
 152
 153   for (const directory of directories) {
 154     const files = await readdir(join(root(), directory))
 155     expect(files).to.have.length.at.least(4)
 156
 157     for (const resolution of [ 240, 360, 480, 720 ]) {
 158       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 159     }
 160   }
 161 }
 162
 163 async function check0PlaylistRedundancies (videoUUID?: string) {
 164   if (!videoUUID) videoUUID = video1Server2UUID
 165
 166   for (const server of servers) {
 167     // With token to avoid issues with video follow constraints
 168     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 169     const video: VideoDetails = res.body
 170
 171     expect(video.streamingPlaylists).to.be.an('array')
 172     expect(video.streamingPlaylists).to.have.lengthOf(1)
 173     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 174   }
 175 }
 176
 177 async function check1PlaylistRedundancies (videoUUID?: string) {
 178   if (!videoUUID) videoUUID = video1Server2UUID
 179
 180   for (const server of servers) {
 181     const res = await getVideo(server.url, videoUUID)
 182     const video: VideoDetails = res.body
 183
 184     expect(video.streamingPlaylists).to.have.lengthOf(1)
 185     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 186
 187     const redundancy = video.streamingPlaylists[0].redundancies[0]
 188
 189     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 190   }
 191
 192   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 193   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 194
 195   const res = await getVideo(servers[0].url, videoUUID)
 196   const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
 197
 198   for (const resolution of [ 240, 360, 480, 720 ]) {
 199     await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
 200   }
 201
 202   const directories = [
 203     'test' + servers[0].internalServerNumber + '/redundancy/hls',
 204     'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
 205   ]
 206
 207   for (const directory of directories) {
 208     const files = await readdir(join(root(), directory, videoUUID))
 209     expect(files).to.have.length.at.least(4)
 210
 211     for (const resolution of [ 240, 360, 480, 720 ]) {
 212       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 213
 214       expect(files.find(f => f === filename)).to.not.be.undefined
 215     }
 216   }
 217 }
 218
 219 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 220   const res = await getStats(servers[0].url)
 221   const data: ServerStats = res.body
 222
 223   expect(data.videosRedundancy).to.have.lengthOf(1)
 224   const stat = data.videosRedundancy[0]
 225
 226   expect(stat.strategy).to.equal(strategy)
 227   expect(stat.totalSize).to.equal(409600)
 228   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
 229   expect(stat.totalVideoFiles).to.equal(4)
 230   expect(stat.totalVideos).to.equal(1)
 231 }
 232
 233 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 234   const res = await getStats(servers[0].url)
 235   const data: ServerStats = res.body
 236
 237   expect(data.videosRedundancy).to.have.lengthOf(1)
 238
 239   const stat = data.videosRedundancy[0]
 240   expect(stat.strategy).to.equal(strategy)
 241   expect(stat.totalSize).to.equal(409600)
 242   expect(stat.totalUsed).to.equal(0)
 243   expect(stat.totalVideoFiles).to.equal(0)
 244   expect(stat.totalVideos).to.equal(0)
 245 }
 246
 247 async function enableRedundancyOnServer1 () {
 248   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 249
 250   const res = await getFollowingListPaginationAndSort({ url: servers[ 0 ].url, start: 0, count: 5, sort: '-createdAt' })
 251   const follows: ActorFollow[] = res.body.data
 252   const server2 = follows.find(f => f.following.host === `localhost:${servers[ 1 ].port}`)
 253   const server3 = follows.find(f => f.following.host === `localhost:${servers[ 2 ].port}`)
 254
 255   expect(server3).to.not.be.undefined
 256   expect(server3.following.hostRedundancyAllowed).to.be.false
 257
 258   expect(server2).to.not.be.undefined
 259   expect(server2.following.hostRedundancyAllowed).to.be.true
 260 }
 261
 262 async function disableRedundancyOnServer1 () {
 263   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
 264
 265   const res = await getFollowingListPaginationAndSort({ url: servers[ 0 ].url, start: 0, count: 5, sort: '-createdAt' })
 266   const follows: ActorFollow[] = res.body.data
 267   const server2 = follows.find(f => f.following.host === `localhost:${servers[ 1 ].port}`)
 268   const server3 = follows.find(f => f.following.host === `localhost:${servers[ 2 ].port}`)
 269
 270   expect(server3).to.not.be.undefined
 271   expect(server3.following.hostRedundancyAllowed).to.be.false
 272
 273   expect(server2).to.not.be.undefined
 274   expect(server2.following.hostRedundancyAllowed).to.be.false
 275 }
 276
 277 describe('Test videos redundancy', function () {
 278
 279   describe('With most-views strategy', function () {
 280     const strategy = 'most-views'
 281
 282     before(function () {
 283       this.timeout(120000)
 284
 285       return flushAndRunServers(strategy)
 286     })
 287
 288     it('Should have 1 webseed on the first video', async function () {
 289       await check1WebSeed()
 290       await check0PlaylistRedundancies()
 291       await checkStatsWith1Webseed(strategy)
 292     })
 293
 294     it('Should enable redundancy on server 1', function () {
 295       return enableRedundancyOnServer1()
 296     })
 297
 298     it('Should have 2 webseeds on the first video', async function () {
 299       this.timeout(80000)
 300
 301       await waitJobs(servers)
 302       await waitUntilLog(servers[0], 'Duplicated ', 5)
 303       await waitJobs(servers)
 304
 305       await check2Webseeds()
 306       await check1PlaylistRedundancies()
 307       await checkStatsWith2Webseed(strategy)
 308     })
 309
 310     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 311       this.timeout(80000)
 312
 313       await disableRedundancyOnServer1()
 314
 315       await waitJobs(servers)
 316       await wait(5000)
 317
 318       await check1WebSeed()
 319       await check0PlaylistRedundancies()
 320
 321       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
 322     })
 323
 324     after(async function () {
 325       return cleanupTests(servers)
 326     })
 327   })
 328
 329   describe('With trending strategy', function () {
 330     const strategy = 'trending'
 331
 332     before(function () {
 333       this.timeout(120000)
 334
 335       return flushAndRunServers(strategy)
 336     })
 337
 338     it('Should have 1 webseed on the first video', async function () {
 339       await check1WebSeed()
 340       await check0PlaylistRedundancies()
 341       await checkStatsWith1Webseed(strategy)
 342     })
 343
 344     it('Should enable redundancy on server 1', function () {
 345       return enableRedundancyOnServer1()
 346     })
 347
 348     it('Should have 2 webseeds on the first video', async function () {
 349       this.timeout(80000)
 350
 351       await waitJobs(servers)
 352       await waitUntilLog(servers[0], 'Duplicated ', 5)
 353       await waitJobs(servers)
 354
 355       await check2Webseeds()
 356       await check1PlaylistRedundancies()
 357       await checkStatsWith2Webseed(strategy)
 358     })
 359
 360     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 361       this.timeout(80000)
 362
 363       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 364
 365       await waitJobs(servers)
 366       await wait(5000)
 367
 368       await check1WebSeed()
 369       await check0PlaylistRedundancies()
 370
 371       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
 372     })
 373
 374     after(async function () {
 375       await cleanupTests(servers)
 376     })
 377   })
 378
 379   describe('With recently added strategy', function () {
 380     const strategy = 'recently-added'
 381
 382     before(function () {
 383       this.timeout(120000)
 384
 385       return flushAndRunServers(strategy, { min_views: 3 })
 386     })
 387
 388     it('Should have 1 webseed on the first video', async function () {
 389       await check1WebSeed()
 390       await check0PlaylistRedundancies()
 391       await checkStatsWith1Webseed(strategy)
 392     })
 393
 394     it('Should enable redundancy on server 1', function () {
 395       return enableRedundancyOnServer1()
 396     })
 397
 398     it('Should still have 1 webseed on the first video', async function () {
 399       this.timeout(80000)
 400
 401       await waitJobs(servers)
 402       await wait(15000)
 403       await waitJobs(servers)
 404
 405       await check1WebSeed()
 406       await check0PlaylistRedundancies()
 407       await checkStatsWith1Webseed(strategy)
 408     })
 409
 410     it('Should view 2 times the first video to have > min_views config', async function () {
 411       this.timeout(80000)
 412
 413       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 414       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 415
 416       await wait(10000)
 417       await waitJobs(servers)
 418     })
 419
 420     it('Should have 2 webseeds on the first video', async function () {
 421       this.timeout(80000)
 422
 423       await waitJobs(servers)
 424       await waitUntilLog(servers[0], 'Duplicated ', 5)
 425       await waitJobs(servers)
 426
 427       await check2Webseeds()
 428       await check1PlaylistRedundancies()
 429       await checkStatsWith2Webseed(strategy)
 430     })
 431
 432     it('Should remove the video and the redundancy files', async function () {
 433       this.timeout(20000)
 434
 435       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 436
 437       await waitJobs(servers)
 438
 439       for (const server of servers) {
 440         await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
 441       }
 442     })
 443
 444     after(async function () {
 445       await cleanupTests(servers)
 446     })
 447   })
 448
 449   describe('Test expiration', function () {
 450     const strategy = 'recently-added'
 451
 452     async function checkContains (servers: ServerInfo[], str: string) {
 453       for (const server of servers) {
 454         const res = await getVideo(server.url, video1Server2UUID)
 455         const video: VideoDetails = res.body
 456
 457         for (const f of video.files) {
 458           expect(f.magnetUri).to.contain(str)
 459         }
 460       }
 461     }
 462
 463     async function checkNotContains (servers: ServerInfo[], str: string) {
 464       for (const server of servers) {
 465         const res = await getVideo(server.url, video1Server2UUID)
 466         const video: VideoDetails = res.body
 467
 468         for (const f of video.files) {
 469           expect(f.magnetUri).to.not.contain(str)
 470         }
 471       }
 472     }
 473
 474     before(async function () {
 475       this.timeout(120000)
 476
 477       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 478
 479       await enableRedundancyOnServer1()
 480     })
 481
 482     it('Should still have 2 webseeds after 10 seconds', async function () {
 483       this.timeout(80000)
 484
 485       await wait(10000)
 486
 487       try {
 488         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 489       } catch {
 490         // Maybe a server deleted a redundancy in the scheduler
 491         await wait(2000)
 492
 493         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 494       }
 495     })
 496
 497     it('Should stop server 1 and expire video redundancy', async function () {
 498       this.timeout(80000)
 499
 500       killallServers([ servers[0] ])
 501
 502       await wait(15000)
 503
 504       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
 505     })
 506
 507     after(async function () {
 508       await cleanupTests(servers)
 509     })
 510   })
 511
 512   describe('Test file replacement', function () {
 513     let video2Server2UUID: string
 514     const strategy = 'recently-added'
 515
 516     before(async function () {
 517       this.timeout(120000)
 518
 519       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 520
 521       await enableRedundancyOnServer1()
 522
 523       await waitJobs(servers)
 524       await waitUntilLog(servers[0], 'Duplicated ', 5)
 525       await waitJobs(servers)
 526
 527       await check2Webseeds()
 528       await check1PlaylistRedundancies()
 529       await checkStatsWith2Webseed(strategy)
 530
 531       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 532       video2Server2UUID = res.body.video.uuid
 533     })
 534
 535     it('Should cache video 2 webseeds on the first video', async function () {
 536       this.timeout(120000)
 537
 538       await waitJobs(servers)
 539
 540       let checked = false
 541
 542       while (checked === false) {
 543         await wait(1000)
 544
 545         try {
 546           await check1WebSeed(video1Server2UUID)
 547           await check0PlaylistRedundancies(video1Server2UUID)
 548           await check2Webseeds(video2Server2UUID)
 549           await check1PlaylistRedundancies(video2Server2UUID)
 550
 551           checked = true
 552         } catch {
 553           checked = false
 554         }
 555       }
 556     })
 557
 558     it('Should disable strategy and remove redundancies', async function () {
 559       this.timeout(80000)
 560
 561       await waitJobs(servers)
 562
 563       killallServers([ servers[ 0 ] ])
 564       await reRunServer(servers[ 0 ], {
 565         redundancy: {
 566           videos: {
 567             check_interval: '1 second',
 568             strategies: []
 569           }
 570         }
 571       })
 572
 573       await waitJobs(servers)
 574
 575       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
 576     })
 577
 578     after(async function () {
 579       await cleanupTests(servers)
 580     })
 581   })
 582 })