server/tests/api/redundancy/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   doubleFollow,
   8   flushAndRunMultipleServers,
   9   getFollowingListPaginationAndSort,
  10   getVideo,
  11   immutableAssign,
  12   killallServers, makeGetRequest,
  13   root,
  14   ServerInfo,
  15   setAccessTokensToServers, unfollow,
  16   uploadVideo,
  17   viewVideo,
  18   wait,
  19   waitUntilLog,
  20   checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer
  21 } from '../../../../shared/utils'
  22 import { waitJobs } from '../../../../shared/utils/server/jobs'
  23
  24 import * as magnetUtil from 'magnet-uri'
  25 import { updateRedundancy } from '../../../../shared/utils/server/redundancy'
  26 import { ActorFollow } from '../../../../shared/models/actors'
  27 import { readdir } from 'fs-extra'
  28 import { join } from 'path'
  29 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  30 import { getStats } from '../../../../shared/utils/server/stats'
  31 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  32
  33 const expect = chai.expect
  34
  35 let servers: ServerInfo[] = []
  36 let video1Server2UUID: string
  37
  38 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  39   const parsed = magnetUtil.decode(file.magnetUri)
  40
  41   for (const ws of baseWebseeds) {
  42     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  43     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  44   }
  45
  46   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  47 }
  48
  49 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  50   const config = {
  51     transcoding: {
  52       hls: {
  53         enabled: true
  54       }
  55     },
  56     redundancy: {
  57       videos: {
  58         check_interval: '5 seconds',
  59         strategies: [
  60           immutableAssign({
  61             min_lifetime: '1 hour',
  62             strategy: strategy,
  63             size: '200KB'
  64           }, additionalParams)
  65         ]
  66       }
  67     }
  68   }
  69   servers = await flushAndRunMultipleServers(3, config)
  70
  71   // Get the access tokens
  72   await setAccessTokensToServers(servers)
  73
  74   {
  75     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  76     video1Server2UUID = res.body.video.uuid
  77
  78     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  79   }
  80
  81   await waitJobs(servers)
  82
  83   // Server 1 and server 2 follow each other
  84   await doubleFollow(servers[ 0 ], servers[ 1 ])
  85   // Server 1 and server 3 follow each other
  86   await doubleFollow(servers[ 0 ], servers[ 2 ])
  87   // Server 2 and server 3 follow each other
  88   await doubleFollow(servers[ 1 ], servers[ 2 ])
  89
  90   await waitJobs(servers)
  91 }
  92
  93 async function check1WebSeed (videoUUID?: string) {
  94   if (!videoUUID) videoUUID = video1Server2UUID
  95
  96   const webseeds = [
  97     'http://localhost:9002/static/webseed/' + videoUUID
  98   ]
  99
 100   for (const server of servers) {
 101     // With token to avoid issues with video follow constraints
 102     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 103
 104     const video: VideoDetails = res.body
 105     for (const f of video.files) {
 106       checkMagnetWebseeds(f, webseeds, server)
 107     }
 108   }
 109 }
 110
 111 async function check2Webseeds (videoUUID?: string) {
 112   if (!videoUUID) videoUUID = video1Server2UUID
 113
 114   const webseeds = [
 115     'http://localhost:9001/static/redundancy/' + videoUUID,
 116     'http://localhost:9002/static/webseed/' + videoUUID
 117   ]
 118
 119   for (const server of servers) {
 120     const res = await getVideo(server.url, videoUUID)
 121
 122     const video: VideoDetails = res.body
 123
 124     for (const file of video.files) {
 125       checkMagnetWebseeds(file, webseeds, server)
 126
 127       await makeGetRequest({
 128         url: servers[0].url,
 129         statusCodeExpected: 200,
 130         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 131         contentType: null
 132       })
 133       await makeGetRequest({
 134         url: servers[1].url,
 135         statusCodeExpected: 200,
 136         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 137         contentType: null
 138       })
 139     }
 140   }
 141
 142   for (const directory of [ 'test1/redundancy', 'test2/videos' ]) {
 143     const files = await readdir(join(root(), directory))
 144     expect(files).to.have.length.at.least(4)
 145
 146     for (const resolution of [ 240, 360, 480, 720 ]) {
 147       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 148     }
 149   }
 150 }
 151
 152 async function check0PlaylistRedundancies (videoUUID?: string) {
 153   if (!videoUUID) videoUUID = video1Server2UUID
 154
 155   for (const server of servers) {
 156     // With token to avoid issues with video follow constraints
 157     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 158     const video: VideoDetails = res.body
 159
 160     expect(video.streamingPlaylists).to.be.an('array')
 161     expect(video.streamingPlaylists).to.have.lengthOf(1)
 162     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 163   }
 164 }
 165
 166 async function check1PlaylistRedundancies (videoUUID?: string) {
 167   if (!videoUUID) videoUUID = video1Server2UUID
 168
 169   for (const server of servers) {
 170     const res = await getVideo(server.url, videoUUID)
 171     const video: VideoDetails = res.body
 172
 173     expect(video.streamingPlaylists).to.have.lengthOf(1)
 174     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 175
 176     const redundancy = video.streamingPlaylists[0].redundancies[0]
 177
 178     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 179   }
 180
 181   await makeGetRequest({
 182     url: servers[0].url,
 183     statusCodeExpected: 200,
 184     path: `/static/redundancy/hls/${videoUUID}/360_000.ts`,
 185     contentType: null
 186   })
 187
 188   for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) {
 189     const files = await readdir(join(root(), directory, videoUUID))
 190     expect(files).to.have.length.at.least(4)
 191
 192     for (const resolution of [ 240, 360, 480, 720 ]) {
 193       expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined
 194       expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined
 195     }
 196   }
 197 }
 198
 199 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 200   const res = await getStats(servers[0].url)
 201   const data: ServerStats = res.body
 202
 203   expect(data.videosRedundancy).to.have.lengthOf(1)
 204   const stat = data.videosRedundancy[0]
 205
 206   expect(stat.strategy).to.equal(strategy)
 207   expect(stat.totalSize).to.equal(204800)
 208   expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
 209   expect(stat.totalVideoFiles).to.equal(4)
 210   expect(stat.totalVideos).to.equal(1)
 211 }
 212
 213 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 214   const res = await getStats(servers[0].url)
 215   const data: ServerStats = res.body
 216
 217   expect(data.videosRedundancy).to.have.lengthOf(1)
 218
 219   const stat = data.videosRedundancy[0]
 220   expect(stat.strategy).to.equal(strategy)
 221   expect(stat.totalSize).to.equal(204800)
 222   expect(stat.totalUsed).to.equal(0)
 223   expect(stat.totalVideoFiles).to.equal(0)
 224   expect(stat.totalVideos).to.equal(0)
 225 }
 226
 227 async function enableRedundancyOnServer1 () {
 228   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 229
 230   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 231   const follows: ActorFollow[] = res.body.data
 232   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 233   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 234
 235   expect(server3).to.not.be.undefined
 236   expect(server3.following.hostRedundancyAllowed).to.be.false
 237
 238   expect(server2).to.not.be.undefined
 239   expect(server2.following.hostRedundancyAllowed).to.be.true
 240 }
 241
 242 async function disableRedundancyOnServer1 () {
 243   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
 244
 245   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 246   const follows: ActorFollow[] = res.body.data
 247   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 248   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 249
 250   expect(server3).to.not.be.undefined
 251   expect(server3.following.hostRedundancyAllowed).to.be.false
 252
 253   expect(server2).to.not.be.undefined
 254   expect(server2.following.hostRedundancyAllowed).to.be.false
 255 }
 256
 257 async function cleanServers () {
 258   killallServers(servers)
 259 }
 260
 261 describe('Test videos redundancy', function () {
 262
 263   describe('With most-views strategy', function () {
 264     const strategy = 'most-views'
 265
 266     before(function () {
 267       this.timeout(120000)
 268
 269       return runServers(strategy)
 270     })
 271
 272     it('Should have 1 webseed on the first video', async function () {
 273       await check1WebSeed()
 274       await check0PlaylistRedundancies()
 275       await checkStatsWith1Webseed(strategy)
 276     })
 277
 278     it('Should enable redundancy on server 1', function () {
 279       return enableRedundancyOnServer1()
 280     })
 281
 282     it('Should have 2 webseeds on the first video', async function () {
 283       this.timeout(80000)
 284
 285       await waitJobs(servers)
 286       await waitUntilLog(servers[0], 'Duplicated ', 5)
 287       await waitJobs(servers)
 288
 289       await check2Webseeds()
 290       await check1PlaylistRedundancies()
 291       await checkStatsWith2Webseed(strategy)
 292     })
 293
 294     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 295       this.timeout(80000)
 296
 297       await disableRedundancyOnServer1()
 298
 299       await waitJobs(servers)
 300       await wait(5000)
 301
 302       await check1WebSeed()
 303       await check0PlaylistRedundancies()
 304
 305       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
 306     })
 307
 308     after(function () {
 309       return cleanServers()
 310     })
 311   })
 312
 313   describe('With trending strategy', function () {
 314     const strategy = 'trending'
 315
 316     before(function () {
 317       this.timeout(120000)
 318
 319       return runServers(strategy)
 320     })
 321
 322     it('Should have 1 webseed on the first video', async function () {
 323       await check1WebSeed()
 324       await check0PlaylistRedundancies()
 325       await checkStatsWith1Webseed(strategy)
 326     })
 327
 328     it('Should enable redundancy on server 1', function () {
 329       return enableRedundancyOnServer1()
 330     })
 331
 332     it('Should have 2 webseeds on the first video', async function () {
 333       this.timeout(80000)
 334
 335       await waitJobs(servers)
 336       await waitUntilLog(servers[0], 'Duplicated ', 5)
 337       await waitJobs(servers)
 338
 339       await check2Webseeds()
 340       await check1PlaylistRedundancies()
 341       await checkStatsWith2Webseed(strategy)
 342     })
 343
 344     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 345       this.timeout(80000)
 346
 347       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 348
 349       await waitJobs(servers)
 350       await wait(5000)
 351
 352       await check1WebSeed()
 353       await check0PlaylistRedundancies()
 354
 355       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 356     })
 357
 358     after(function () {
 359       return cleanServers()
 360     })
 361   })
 362
 363   describe('With recently added strategy', function () {
 364     const strategy = 'recently-added'
 365
 366     before(function () {
 367       this.timeout(120000)
 368
 369       return runServers(strategy, { min_views: 3 })
 370     })
 371
 372     it('Should have 1 webseed on the first video', async function () {
 373       await check1WebSeed()
 374       await check0PlaylistRedundancies()
 375       await checkStatsWith1Webseed(strategy)
 376     })
 377
 378     it('Should enable redundancy on server 1', function () {
 379       return enableRedundancyOnServer1()
 380     })
 381
 382     it('Should still have 1 webseed on the first video', async function () {
 383       this.timeout(80000)
 384
 385       await waitJobs(servers)
 386       await wait(15000)
 387       await waitJobs(servers)
 388
 389       await check1WebSeed()
 390       await check0PlaylistRedundancies()
 391       await checkStatsWith1Webseed(strategy)
 392     })
 393
 394     it('Should view 2 times the first video to have > min_views config', async function () {
 395       this.timeout(80000)
 396
 397       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 398       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 399
 400       await wait(10000)
 401       await waitJobs(servers)
 402     })
 403
 404     it('Should have 2 webseeds on the first video', async function () {
 405       this.timeout(80000)
 406
 407       await waitJobs(servers)
 408       await waitUntilLog(servers[0], 'Duplicated ', 5)
 409       await waitJobs(servers)
 410
 411       await check2Webseeds()
 412       await check1PlaylistRedundancies()
 413       await checkStatsWith2Webseed(strategy)
 414     })
 415
 416     it('Should remove the video and the redundancy files', async function () {
 417       this.timeout(20000)
 418
 419       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 420
 421       await waitJobs(servers)
 422
 423       for (const server of servers) {
 424         await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber)
 425       }
 426     })
 427
 428     after(function () {
 429       return cleanServers()
 430     })
 431   })
 432
 433   describe('Test expiration', function () {
 434     const strategy = 'recently-added'
 435
 436     async function checkContains (servers: ServerInfo[], str: string) {
 437       for (const server of servers) {
 438         const res = await getVideo(server.url, video1Server2UUID)
 439         const video: VideoDetails = res.body
 440
 441         for (const f of video.files) {
 442           expect(f.magnetUri).to.contain(str)
 443         }
 444       }
 445     }
 446
 447     async function checkNotContains (servers: ServerInfo[], str: string) {
 448       for (const server of servers) {
 449         const res = await getVideo(server.url, video1Server2UUID)
 450         const video: VideoDetails = res.body
 451
 452         for (const f of video.files) {
 453           expect(f.magnetUri).to.not.contain(str)
 454         }
 455       }
 456     }
 457
 458     before(async function () {
 459       this.timeout(120000)
 460
 461       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 462
 463       await enableRedundancyOnServer1()
 464     })
 465
 466     it('Should still have 2 webseeds after 10 seconds', async function () {
 467       this.timeout(80000)
 468
 469       await wait(10000)
 470
 471       try {
 472         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 473       } catch {
 474         // Maybe a server deleted a redundancy in the scheduler
 475         await wait(2000)
 476
 477         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 478       }
 479     })
 480
 481     it('Should stop server 1 and expire video redundancy', async function () {
 482       this.timeout(80000)
 483
 484       killallServers([ servers[0] ])
 485
 486       await wait(15000)
 487
 488       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 489     })
 490
 491     after(function () {
 492       return killallServers([ servers[1], servers[2] ])
 493     })
 494   })
 495
 496   describe('Test file replacement', function () {
 497     let video2Server2UUID: string
 498     const strategy = 'recently-added'
 499
 500     before(async function () {
 501       this.timeout(120000)
 502
 503       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 504
 505       await enableRedundancyOnServer1()
 506
 507       await waitJobs(servers)
 508       await waitUntilLog(servers[0], 'Duplicated ', 5)
 509       await waitJobs(servers)
 510
 511       await check2Webseeds()
 512       await check1PlaylistRedundancies()
 513       await checkStatsWith2Webseed(strategy)
 514
 515       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 516       video2Server2UUID = res.body.video.uuid
 517     })
 518
 519     it('Should cache video 2 webseeds on the first video', async function () {
 520       this.timeout(120000)
 521
 522       await waitJobs(servers)
 523
 524       let checked = false
 525
 526       while (checked === false) {
 527         await wait(1000)
 528
 529         try {
 530           await check1WebSeed(video1Server2UUID)
 531           await check0PlaylistRedundancies(video1Server2UUID)
 532           await check2Webseeds(video2Server2UUID)
 533           await check1PlaylistRedundancies(video2Server2UUID)
 534
 535           checked = true
 536         } catch {
 537           checked = false
 538         }
 539       }
 540     })
 541
 542     it('Should disable strategy and remove redundancies', async function () {
 543       this.timeout(80000)
 544
 545       await waitJobs(servers)
 546
 547       killallServers([ servers[ 0 ] ])
 548       await reRunServer(servers[ 0 ], {
 549         redundancy: {
 550           videos: {
 551             check_interval: '1 second',
 552             strategies: []
 553           }
 554         }
 555       })
 556
 557       await waitJobs(servers)
 558
 559       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
 560     })
 561
 562     after(function () {
 563       return cleanServers()
 564     })
 565   })
 566 })