server/tests/api/redundancy/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   checkSegmentHash,
   8   checkVideoFilesWereRemoved, cleanupTests,
   9   doubleFollow,
  10   flushAndRunMultipleServers,
  11   getFollowingListPaginationAndSort,
  12   getVideo,
  13   getVideoWithToken,
  14   immutableAssign,
  15   killallServers,
  16   makeGetRequest,
  17   removeVideo,
  18   reRunServer,
  19   root,
  20   ServerInfo,
  21   setAccessTokensToServers,
  22   unfollow,
  23   uploadVideo,
  24   viewVideo,
  25   wait,
  26   waitUntilLog
  27 } from '../../../../shared/extra-utils'
  28 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
  29
  30 import * as magnetUtil from 'magnet-uri'
  31 import { updateRedundancy } from '../../../../shared/extra-utils/server/redundancy'
  32 import { ActorFollow } from '../../../../shared/models/actors'
  33 import { readdir } from 'fs-extra'
  34 import { join } from 'path'
  35 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  36 import { getStats } from '../../../../shared/extra-utils/server/stats'
  37 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  38
  39 const expect = chai.expect
  40
  41 let servers: ServerInfo[] = []
  42 let video1Server2UUID: string
  43
  44 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  45   const parsed = magnetUtil.decode(file.magnetUri)
  46
  47   for (const ws of baseWebseeds) {
  48     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  49     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  50   }
  51
  52   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  53 }
  54
  55 async function flushAndRunServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  56   const config = {
  57     transcoding: {
  58       hls: {
  59         enabled: true
  60       }
  61     },
  62     redundancy: {
  63       videos: {
  64         check_interval: '5 seconds',
  65         strategies: [
  66           immutableAssign({
  67             min_lifetime: '1 hour',
  68             strategy: strategy,
  69             size: '200KB'
  70           }, additionalParams)
  71         ]
  72       }
  73     }
  74   }
  75   servers = await flushAndRunMultipleServers(3, config)
  76
  77   // Get the access tokens
  78   await setAccessTokensToServers(servers)
  79
  80   {
  81     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  82     video1Server2UUID = res.body.video.uuid
  83
  84     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  85   }
  86
  87   await waitJobs(servers)
  88
  89   // Server 1 and server 2 follow each other
  90   await doubleFollow(servers[ 0 ], servers[ 1 ])
  91   // Server 1 and server 3 follow each other
  92   await doubleFollow(servers[ 0 ], servers[ 2 ])
  93   // Server 2 and server 3 follow each other
  94   await doubleFollow(servers[ 1 ], servers[ 2 ])
  95
  96   await waitJobs(servers)
  97 }
  98
  99 async function check1WebSeed (videoUUID?: string) {
 100   if (!videoUUID) videoUUID = video1Server2UUID
 101
 102   const webseeds = [
 103     'http://localhost:9002/static/webseed/' + videoUUID
 104   ]
 105
 106   for (const server of servers) {
 107     // With token to avoid issues with video follow constraints
 108     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 109
 110     const video: VideoDetails = res.body
 111     for (const f of video.files) {
 112       checkMagnetWebseeds(f, webseeds, server)
 113     }
 114   }
 115 }
 116
 117 async function check2Webseeds (videoUUID?: string) {
 118   if (!videoUUID) videoUUID = video1Server2UUID
 119
 120   const webseeds = [
 121     'http://localhost:9001/static/redundancy/' + videoUUID,
 122     'http://localhost:9002/static/webseed/' + videoUUID
 123   ]
 124
 125   for (const server of servers) {
 126     const res = await getVideo(server.url, videoUUID)
 127
 128     const video: VideoDetails = res.body
 129
 130     for (const file of video.files) {
 131       checkMagnetWebseeds(file, webseeds, server)
 132
 133       await makeGetRequest({
 134         url: servers[0].url,
 135         statusCodeExpected: 200,
 136         path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
 137         contentType: null
 138       })
 139       await makeGetRequest({
 140         url: servers[1].url,
 141         statusCodeExpected: 200,
 142         path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
 143         contentType: null
 144       })
 145     }
 146   }
 147
 148   for (const directory of [ 'test1/redundancy', 'test2/videos' ]) {
 149     const files = await readdir(join(root(), directory))
 150     expect(files).to.have.length.at.least(4)
 151
 152     for (const resolution of [ 240, 360, 480, 720 ]) {
 153       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 154     }
 155   }
 156 }
 157
 158 async function check0PlaylistRedundancies (videoUUID?: string) {
 159   if (!videoUUID) videoUUID = video1Server2UUID
 160
 161   for (const server of servers) {
 162     // With token to avoid issues with video follow constraints
 163     const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 164     const video: VideoDetails = res.body
 165
 166     expect(video.streamingPlaylists).to.be.an('array')
 167     expect(video.streamingPlaylists).to.have.lengthOf(1)
 168     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
 169   }
 170 }
 171
 172 async function check1PlaylistRedundancies (videoUUID?: string) {
 173   if (!videoUUID) videoUUID = video1Server2UUID
 174
 175   for (const server of servers) {
 176     const res = await getVideo(server.url, videoUUID)
 177     const video: VideoDetails = res.body
 178
 179     expect(video.streamingPlaylists).to.have.lengthOf(1)
 180     expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
 181
 182     const redundancy = video.streamingPlaylists[0].redundancies[0]
 183
 184     expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
 185   }
 186
 187   const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
 188   const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
 189
 190   const res = await getVideo(servers[0].url, videoUUID)
 191   const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
 192
 193   for (const resolution of [ 240, 360, 480, 720 ]) {
 194     await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
 195   }
 196
 197   for (const directory of [ 'test1/redundancy/hls', 'test2/streaming-playlists/hls' ]) {
 198     const files = await readdir(join(root(), directory, videoUUID))
 199     expect(files).to.have.length.at.least(4)
 200
 201     for (const resolution of [ 240, 360, 480, 720 ]) {
 202       const filename = `${videoUUID}-${resolution}-fragmented.mp4`
 203
 204       expect(files.find(f => f === filename)).to.not.be.undefined
 205     }
 206   }
 207 }
 208
 209 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 210   const res = await getStats(servers[0].url)
 211   const data: ServerStats = res.body
 212
 213   expect(data.videosRedundancy).to.have.lengthOf(1)
 214   const stat = data.videosRedundancy[0]
 215
 216   expect(stat.strategy).to.equal(strategy)
 217   expect(stat.totalSize).to.equal(204800)
 218   expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
 219   expect(stat.totalVideoFiles).to.equal(4)
 220   expect(stat.totalVideos).to.equal(1)
 221 }
 222
 223 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 224   const res = await getStats(servers[0].url)
 225   const data: ServerStats = res.body
 226
 227   expect(data.videosRedundancy).to.have.lengthOf(1)
 228
 229   const stat = data.videosRedundancy[0]
 230   expect(stat.strategy).to.equal(strategy)
 231   expect(stat.totalSize).to.equal(204800)
 232   expect(stat.totalUsed).to.equal(0)
 233   expect(stat.totalVideoFiles).to.equal(0)
 234   expect(stat.totalVideos).to.equal(0)
 235 }
 236
 237 async function enableRedundancyOnServer1 () {
 238   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 239
 240   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 241   const follows: ActorFollow[] = res.body.data
 242   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 243   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 244
 245   expect(server3).to.not.be.undefined
 246   expect(server3.following.hostRedundancyAllowed).to.be.false
 247
 248   expect(server2).to.not.be.undefined
 249   expect(server2.following.hostRedundancyAllowed).to.be.true
 250 }
 251
 252 async function disableRedundancyOnServer1 () {
 253   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
 254
 255   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 256   const follows: ActorFollow[] = res.body.data
 257   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 258   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 259
 260   expect(server3).to.not.be.undefined
 261   expect(server3.following.hostRedundancyAllowed).to.be.false
 262
 263   expect(server2).to.not.be.undefined
 264   expect(server2.following.hostRedundancyAllowed).to.be.false
 265 }
 266
 267 describe('Test videos redundancy', function () {
 268
 269   describe('With most-views strategy', function () {
 270     const strategy = 'most-views'
 271
 272     before(function () {
 273       this.timeout(120000)
 274
 275       return flushAndRunServers(strategy)
 276     })
 277
 278     it('Should have 1 webseed on the first video', async function () {
 279       await check1WebSeed()
 280       await check0PlaylistRedundancies()
 281       await checkStatsWith1Webseed(strategy)
 282     })
 283
 284     it('Should enable redundancy on server 1', function () {
 285       return enableRedundancyOnServer1()
 286     })
 287
 288     it('Should have 2 webseeds on the first video', async function () {
 289       this.timeout(80000)
 290
 291       await waitJobs(servers)
 292       await waitUntilLog(servers[0], 'Duplicated ', 5)
 293       await waitJobs(servers)
 294
 295       await check2Webseeds()
 296       await check1PlaylistRedundancies()
 297       await checkStatsWith2Webseed(strategy)
 298     })
 299
 300     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 301       this.timeout(80000)
 302
 303       await disableRedundancyOnServer1()
 304
 305       await waitJobs(servers)
 306       await wait(5000)
 307
 308       await check1WebSeed()
 309       await check0PlaylistRedundancies()
 310
 311       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
 312     })
 313
 314     after(async function () {
 315       return cleanupTests(servers)
 316     })
 317   })
 318
 319   describe('With trending strategy', function () {
 320     const strategy = 'trending'
 321
 322     before(function () {
 323       this.timeout(120000)
 324
 325       return flushAndRunServers(strategy)
 326     })
 327
 328     it('Should have 1 webseed on the first video', async function () {
 329       await check1WebSeed()
 330       await check0PlaylistRedundancies()
 331       await checkStatsWith1Webseed(strategy)
 332     })
 333
 334     it('Should enable redundancy on server 1', function () {
 335       return enableRedundancyOnServer1()
 336     })
 337
 338     it('Should have 2 webseeds on the first video', async function () {
 339       this.timeout(80000)
 340
 341       await waitJobs(servers)
 342       await waitUntilLog(servers[0], 'Duplicated ', 5)
 343       await waitJobs(servers)
 344
 345       await check2Webseeds()
 346       await check1PlaylistRedundancies()
 347       await checkStatsWith2Webseed(strategy)
 348     })
 349
 350     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 351       this.timeout(80000)
 352
 353       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 354
 355       await waitJobs(servers)
 356       await wait(5000)
 357
 358       await check1WebSeed()
 359       await check0PlaylistRedundancies()
 360
 361       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 362     })
 363
 364     after(async function () {
 365       await cleanupTests(servers)
 366     })
 367   })
 368
 369   describe('With recently added strategy', function () {
 370     const strategy = 'recently-added'
 371
 372     before(function () {
 373       this.timeout(120000)
 374
 375       return flushAndRunServers(strategy, { min_views: 3 })
 376     })
 377
 378     it('Should have 1 webseed on the first video', async function () {
 379       await check1WebSeed()
 380       await check0PlaylistRedundancies()
 381       await checkStatsWith1Webseed(strategy)
 382     })
 383
 384     it('Should enable redundancy on server 1', function () {
 385       return enableRedundancyOnServer1()
 386     })
 387
 388     it('Should still have 1 webseed on the first video', async function () {
 389       this.timeout(80000)
 390
 391       await waitJobs(servers)
 392       await wait(15000)
 393       await waitJobs(servers)
 394
 395       await check1WebSeed()
 396       await check0PlaylistRedundancies()
 397       await checkStatsWith1Webseed(strategy)
 398     })
 399
 400     it('Should view 2 times the first video to have > min_views config', async function () {
 401       this.timeout(80000)
 402
 403       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 404       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 405
 406       await wait(10000)
 407       await waitJobs(servers)
 408     })
 409
 410     it('Should have 2 webseeds on the first video', async function () {
 411       this.timeout(80000)
 412
 413       await waitJobs(servers)
 414       await waitUntilLog(servers[0], 'Duplicated ', 5)
 415       await waitJobs(servers)
 416
 417       await check2Webseeds()
 418       await check1PlaylistRedundancies()
 419       await checkStatsWith2Webseed(strategy)
 420     })
 421
 422     it('Should remove the video and the redundancy files', async function () {
 423       this.timeout(20000)
 424
 425       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 426
 427       await waitJobs(servers)
 428
 429       for (const server of servers) {
 430         await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber)
 431       }
 432     })
 433
 434     after(async function () {
 435       await cleanupTests(servers)
 436     })
 437   })
 438
 439   describe('Test expiration', function () {
 440     const strategy = 'recently-added'
 441
 442     async function checkContains (servers: ServerInfo[], str: string) {
 443       for (const server of servers) {
 444         const res = await getVideo(server.url, video1Server2UUID)
 445         const video: VideoDetails = res.body
 446
 447         for (const f of video.files) {
 448           expect(f.magnetUri).to.contain(str)
 449         }
 450       }
 451     }
 452
 453     async function checkNotContains (servers: ServerInfo[], str: string) {
 454       for (const server of servers) {
 455         const res = await getVideo(server.url, video1Server2UUID)
 456         const video: VideoDetails = res.body
 457
 458         for (const f of video.files) {
 459           expect(f.magnetUri).to.not.contain(str)
 460         }
 461       }
 462     }
 463
 464     before(async function () {
 465       this.timeout(120000)
 466
 467       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 468
 469       await enableRedundancyOnServer1()
 470     })
 471
 472     it('Should still have 2 webseeds after 10 seconds', async function () {
 473       this.timeout(80000)
 474
 475       await wait(10000)
 476
 477       try {
 478         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 479       } catch {
 480         // Maybe a server deleted a redundancy in the scheduler
 481         await wait(2000)
 482
 483         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 484       }
 485     })
 486
 487     it('Should stop server 1 and expire video redundancy', async function () {
 488       this.timeout(80000)
 489
 490       killallServers([ servers[0] ])
 491
 492       await wait(15000)
 493
 494       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 495     })
 496
 497     after(async function () {
 498       await cleanupTests(servers)
 499     })
 500   })
 501
 502   describe('Test file replacement', function () {
 503     let video2Server2UUID: string
 504     const strategy = 'recently-added'
 505
 506     before(async function () {
 507       this.timeout(120000)
 508
 509       await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 510
 511       await enableRedundancyOnServer1()
 512
 513       await waitJobs(servers)
 514       await waitUntilLog(servers[0], 'Duplicated ', 5)
 515       await waitJobs(servers)
 516
 517       await check2Webseeds()
 518       await check1PlaylistRedundancies()
 519       await checkStatsWith2Webseed(strategy)
 520
 521       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 522       video2Server2UUID = res.body.video.uuid
 523     })
 524
 525     it('Should cache video 2 webseeds on the first video', async function () {
 526       this.timeout(120000)
 527
 528       await waitJobs(servers)
 529
 530       let checked = false
 531
 532       while (checked === false) {
 533         await wait(1000)
 534
 535         try {
 536           await check1WebSeed(video1Server2UUID)
 537           await check0PlaylistRedundancies(video1Server2UUID)
 538           await check2Webseeds(video2Server2UUID)
 539           await check1PlaylistRedundancies(video2Server2UUID)
 540
 541           checked = true
 542         } catch {
 543           checked = false
 544         }
 545       }
 546     })
 547
 548     it('Should disable strategy and remove redundancies', async function () {
 549       this.timeout(80000)
 550
 551       await waitJobs(servers)
 552
 553       killallServers([ servers[ 0 ] ])
 554       await reRunServer(servers[ 0 ], {
 555         redundancy: {
 556           videos: {
 557             check_interval: '1 second',
 558             strategies: []
 559           }
 560         }
 561       })
 562
 563       await waitJobs(servers)
 564
 565       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
 566     })
 567
 568     after(async function () {
 569       await cleanupTests(servers)
 570     })
 571   })
 572 })