server/tests/api/server/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   doubleFollow,
   8   flushAndRunMultipleServers,
   9   getFollowingListPaginationAndSort,
  10   getVideo,
  11   immutableAssign,
  12   killallServers, makeGetRequest,
  13   root,
  14   ServerInfo,
  15   setAccessTokensToServers, unfollow,
  16   uploadVideo,
  17   viewVideo,
  18   wait
  19 } from '../../utils'
  20 import { waitJobs } from '../../utils/server/jobs'
  21 import * as magnetUtil from 'magnet-uri'
  22 import { updateRedundancy } from '../../utils/server/redundancy'
  23 import { ActorFollow } from '../../../../shared/models/actors'
  24 import { readdir } from 'fs-extra'
  25 import { join } from 'path'
  26 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  27 import { getStats } from '../../utils/server/stats'
  28 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  29
  30 const expect = chai.expect
  31
  32 let servers: ServerInfo[] = []
  33 let video1Server2UUID: string
  34
  35 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  36   const parsed = magnetUtil.decode(file.magnetUri)
  37
  38   for (const ws of baseWebseeds) {
  39     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  40     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  41   }
  42
  43   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  44 }
  45
  46 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  47   const config = {
  48     redundancy: {
  49       videos: {
  50         check_interval: '5 seconds',
  51         strategies: [
  52           immutableAssign({
  53             min_lifetime: '1 hour',
  54             strategy: strategy,
  55             size: '100KB'
  56           }, additionalParams)
  57         ]
  58       }
  59     }
  60   }
  61   servers = await flushAndRunMultipleServers(3, config)
  62
  63   // Get the access tokens
  64   await setAccessTokensToServers(servers)
  65
  66   {
  67     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  68     video1Server2UUID = res.body.video.uuid
  69
  70     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  71   }
  72
  73   await waitJobs(servers)
  74
  75   // Server 1 and server 2 follow each other
  76   await doubleFollow(servers[ 0 ], servers[ 1 ])
  77   // Server 1 and server 3 follow each other
  78   await doubleFollow(servers[ 0 ], servers[ 2 ])
  79   // Server 2 and server 3 follow each other
  80   await doubleFollow(servers[ 1 ], servers[ 2 ])
  81
  82   await waitJobs(servers)
  83 }
  84
  85 async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
  86   if (!videoUUID) videoUUID = video1Server2UUID
  87
  88   const webseeds = [
  89     'http://localhost:9002/static/webseed/' + videoUUID
  90   ]
  91
  92   for (const server of servers) {
  93     {
  94       const res = await getVideo(server.url, videoUUID)
  95
  96       const video: VideoDetails = res.body
  97       for (const f of video.files) {
  98         checkMagnetWebseeds(f, webseeds, server)
  99       }
 100     }
 101   }
 102 }
 103
 104 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 105   const res = await getStats(servers[0].url)
 106   const data: ServerStats = res.body
 107
 108   expect(data.videosRedundancy).to.have.lengthOf(1)
 109   const stat = data.videosRedundancy[0]
 110
 111   expect(stat.strategy).to.equal(strategy)
 112   expect(stat.totalSize).to.equal(102400)
 113   expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
 114   expect(stat.totalVideoFiles).to.equal(4)
 115   expect(stat.totalVideos).to.equal(1)
 116 }
 117
 118 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 119   const res = await getStats(servers[0].url)
 120   const data: ServerStats = res.body
 121
 122   expect(data.videosRedundancy).to.have.lengthOf(1)
 123
 124   const stat = data.videosRedundancy[0]
 125   expect(stat.strategy).to.equal(strategy)
 126   expect(stat.totalSize).to.equal(102400)
 127   expect(stat.totalUsed).to.equal(0)
 128   expect(stat.totalVideoFiles).to.equal(0)
 129   expect(stat.totalVideos).to.equal(0)
 130 }
 131
 132 async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
 133   if (!videoUUID) videoUUID = video1Server2UUID
 134
 135   const webseeds = [
 136     'http://localhost:9001/static/webseed/' + videoUUID,
 137     'http://localhost:9002/static/webseed/' + videoUUID
 138   ]
 139
 140   for (const server of servers) {
 141     const res = await getVideo(server.url, videoUUID)
 142
 143     const video: VideoDetails = res.body
 144
 145     for (const file of video.files) {
 146       checkMagnetWebseeds(file, webseeds, server)
 147
 148       // Only servers 1 and 2 have the video
 149       if (server.serverNumber !== 3) {
 150         await makeGetRequest({
 151           url: server.url,
 152           statusCodeExpected: 200,
 153           path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
 154           contentType: null
 155         })
 156       }
 157     }
 158   }
 159
 160   for (const directory of [ 'test1', 'test2' ]) {
 161     const files = await readdir(join(root(), directory, 'videos'))
 162     expect(files).to.have.length.at.least(4)
 163
 164     for (const resolution of [ 240, 360, 480, 720 ]) {
 165       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 166     }
 167   }
 168 }
 169
 170 async function enableRedundancyOnServer1 () {
 171   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 172
 173   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 174   const follows: ActorFollow[] = res.body.data
 175   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 176   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 177
 178   expect(server3).to.not.be.undefined
 179   expect(server3.following.hostRedundancyAllowed).to.be.false
 180
 181   expect(server2).to.not.be.undefined
 182   expect(server2.following.hostRedundancyAllowed).to.be.true
 183 }
 184
 185 async function disableRedundancyOnServer1 () {
 186   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
 187
 188   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 189   const follows: ActorFollow[] = res.body.data
 190   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 191   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 192
 193   expect(server3).to.not.be.undefined
 194   expect(server3.following.hostRedundancyAllowed).to.be.false
 195
 196   expect(server2).to.not.be.undefined
 197   expect(server2.following.hostRedundancyAllowed).to.be.false
 198 }
 199
 200 async function cleanServers () {
 201   killallServers(servers)
 202 }
 203
 204 describe('Test videos redundancy', function () {
 205
 206   describe('With most-views strategy', function () {
 207     const strategy = 'most-views'
 208
 209     before(function () {
 210       this.timeout(120000)
 211
 212       return runServers(strategy)
 213     })
 214
 215     it('Should have 1 webseed on the first video', async function () {
 216       await check1WebSeed(strategy)
 217       await checkStatsWith1Webseed(strategy)
 218     })
 219
 220     it('Should enable redundancy on server 1', function () {
 221       return enableRedundancyOnServer1()
 222     })
 223
 224     it('Should have 2 webseed on the first video', async function () {
 225       this.timeout(40000)
 226
 227       await waitJobs(servers)
 228       await wait(15000)
 229       await waitJobs(servers)
 230
 231       await check2Webseeds(strategy)
 232       await checkStatsWith2Webseed(strategy)
 233     })
 234
 235     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 236       this.timeout(40000)
 237
 238       await disableRedundancyOnServer1()
 239
 240       await waitJobs(servers)
 241       await wait(5000)
 242
 243       await check1WebSeed(strategy)
 244     })
 245
 246     after(function () {
 247       return cleanServers()
 248     })
 249   })
 250
 251   describe('With trending strategy', function () {
 252     const strategy = 'trending'
 253
 254     before(function () {
 255       this.timeout(120000)
 256
 257       return runServers(strategy)
 258     })
 259
 260     it('Should have 1 webseed on the first video', async function () {
 261       await check1WebSeed(strategy)
 262       await checkStatsWith1Webseed(strategy)
 263     })
 264
 265     it('Should enable redundancy on server 1', function () {
 266       return enableRedundancyOnServer1()
 267     })
 268
 269     it('Should have 2 webseed on the first video', async function () {
 270       this.timeout(40000)
 271
 272       await waitJobs(servers)
 273       await wait(15000)
 274       await waitJobs(servers)
 275
 276       await check2Webseeds(strategy)
 277       await checkStatsWith2Webseed(strategy)
 278     })
 279
 280     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 281       this.timeout(40000)
 282
 283       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 284
 285       await waitJobs(servers)
 286       await wait(5000)
 287
 288       await check1WebSeed(strategy)
 289     })
 290
 291     after(function () {
 292       return cleanServers()
 293     })
 294   })
 295
 296   describe('With recently added strategy', function () {
 297     const strategy = 'recently-added'
 298
 299     before(function () {
 300       this.timeout(120000)
 301
 302       return runServers(strategy, { min_views: 3 })
 303     })
 304
 305     it('Should have 1 webseed on the first video', async function () {
 306       await check1WebSeed(strategy)
 307       await checkStatsWith1Webseed(strategy)
 308     })
 309
 310     it('Should enable redundancy on server 1', function () {
 311       return enableRedundancyOnServer1()
 312     })
 313
 314     it('Should still have 1 webseed on the first video', async function () {
 315       this.timeout(40000)
 316
 317       await waitJobs(servers)
 318       await wait(15000)
 319       await waitJobs(servers)
 320
 321       await check1WebSeed(strategy)
 322       await checkStatsWith1Webseed(strategy)
 323     })
 324
 325     it('Should view 2 times the first video to have > min_views config', async function () {
 326       this.timeout(40000)
 327
 328       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 329       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 330
 331       await wait(10000)
 332       await waitJobs(servers)
 333     })
 334
 335     it('Should have 2 webseed on the first video', async function () {
 336       this.timeout(40000)
 337
 338       await waitJobs(servers)
 339       await wait(15000)
 340       await waitJobs(servers)
 341
 342       await check2Webseeds(strategy)
 343       await checkStatsWith2Webseed(strategy)
 344     })
 345
 346     after(function () {
 347       return cleanServers()
 348     })
 349   })
 350
 351   describe('Test expiration', function () {
 352     const strategy = 'recently-added'
 353
 354     async function checkContains (servers: ServerInfo[], str: string) {
 355       for (const server of servers) {
 356         const res = await getVideo(server.url, video1Server2UUID)
 357         const video: VideoDetails = res.body
 358
 359         for (const f of video.files) {
 360           expect(f.magnetUri).to.contain(str)
 361         }
 362       }
 363     }
 364
 365     async function checkNotContains (servers: ServerInfo[], str: string) {
 366       for (const server of servers) {
 367         const res = await getVideo(server.url, video1Server2UUID)
 368         const video: VideoDetails = res.body
 369
 370         for (const f of video.files) {
 371           expect(f.magnetUri).to.not.contain(str)
 372         }
 373       }
 374     }
 375
 376     before(async function () {
 377       this.timeout(120000)
 378
 379       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 380
 381       await enableRedundancyOnServer1()
 382     })
 383
 384     it('Should still have 2 webseeds after 10 seconds', async function () {
 385       this.timeout(40000)
 386
 387       await wait(10000)
 388
 389       try {
 390         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 391       } catch {
 392         // Maybe a server deleted a redundancy in the scheduler
 393         await wait(2000)
 394
 395         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 396       }
 397     })
 398
 399     it('Should stop server 1 and expire video redundancy', async function () {
 400       this.timeout(40000)
 401
 402       killallServers([ servers[0] ])
 403
 404       await wait(10000)
 405
 406       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 407     })
 408
 409     after(function () {
 410       return killallServers([ servers[1], servers[2] ])
 411     })
 412   })
 413
 414   describe('Test file replacement', function () {
 415     let video2Server2UUID: string
 416     const strategy = 'recently-added'
 417
 418     before(async function () {
 419       this.timeout(120000)
 420
 421       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 422
 423       await enableRedundancyOnServer1()
 424
 425       await waitJobs(servers)
 426       await wait(5000)
 427       await waitJobs(servers)
 428
 429       await check2Webseeds(strategy)
 430       await checkStatsWith2Webseed(strategy)
 431
 432       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 433       video2Server2UUID = res.body.video.uuid
 434     })
 435
 436     it('Should cache video 2 webseed on the first video', async function () {
 437       this.timeout(40000)
 438       this.retries(3)
 439
 440       await waitJobs(servers)
 441
 442       await wait(7000)
 443
 444       await check1WebSeed(strategy, video1Server2UUID)
 445       await check2Webseeds(strategy, video2Server2UUID)
 446     })
 447
 448     after(function () {
 449       return cleanServers()
 450     })
 451   })
 452 })