server/tests/api/server/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   doubleFollow,
   8   flushAndRunMultipleServers,
   9   getFollowingListPaginationAndSort,
  10   getVideo,
  11   immutableAssign,
  12   killallServers, makeGetRequest,
  13   root,
  14   ServerInfo,
  15   setAccessTokensToServers,
  16   uploadVideo,
  17   viewVideo,
  18   wait
  19 } from '../../utils'
  20 import { waitJobs } from '../../utils/server/jobs'
  21 import * as magnetUtil from 'magnet-uri'
  22 import { updateRedundancy } from '../../utils/server/redundancy'
  23 import { ActorFollow } from '../../../../shared/models/actors'
  24 import { readdir } from 'fs-extra'
  25 import { join } from 'path'
  26 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  27 import { getStats } from '../../utils/server/stats'
  28 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  29
  30 const expect = chai.expect
  31
  32 let servers: ServerInfo[] = []
  33 let video1Server2UUID: string
  34
  35 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  36   const parsed = magnetUtil.decode(file.magnetUri)
  37
  38   for (const ws of baseWebseeds) {
  39     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  40     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  41   }
  42 }
  43
  44 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  45   const config = {
  46     redundancy: {
  47       videos: {
  48         check_interval: '5 seconds',
  49         strategies: [
  50           immutableAssign({
  51             min_lifetime: '1 hour',
  52             strategy: strategy,
  53             size: '100KB'
  54           }, additionalParams)
  55         ]
  56       }
  57     }
  58   }
  59   servers = await flushAndRunMultipleServers(3, config)
  60
  61   // Get the access tokens
  62   await setAccessTokensToServers(servers)
  63
  64   {
  65     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  66     video1Server2UUID = res.body.video.uuid
  67
  68     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  69   }
  70
  71   await waitJobs(servers)
  72
  73   // Server 1 and server 2 follow each other
  74   await doubleFollow(servers[ 0 ], servers[ 1 ])
  75   // Server 1 and server 3 follow each other
  76   await doubleFollow(servers[ 0 ], servers[ 2 ])
  77   // Server 2 and server 3 follow each other
  78   await doubleFollow(servers[ 1 ], servers[ 2 ])
  79
  80   await waitJobs(servers)
  81 }
  82
  83 async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
  84   if (!videoUUID) videoUUID = video1Server2UUID
  85
  86   const webseeds = [
  87     'http://localhost:9002/static/webseed/' + videoUUID
  88   ]
  89
  90   for (const server of servers) {
  91     {
  92       const res = await getVideo(server.url, videoUUID)
  93
  94       const video: VideoDetails = res.body
  95       for (const f of video.files) {
  96         checkMagnetWebseeds(f, webseeds, server)
  97       }
  98     }
  99   }
 100 }
 101
 102 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 103   const res = await getStats(servers[0].url)
 104   const data: ServerStats = res.body
 105
 106   expect(data.videosRedundancy).to.have.lengthOf(1)
 107   const stat = data.videosRedundancy[0]
 108
 109   expect(stat.strategy).to.equal(strategy)
 110   expect(stat.totalSize).to.equal(102400)
 111   expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
 112   expect(stat.totalVideoFiles).to.equal(4)
 113   expect(stat.totalVideos).to.equal(1)
 114 }
 115
 116 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 117   const res = await getStats(servers[0].url)
 118   const data: ServerStats = res.body
 119
 120   expect(data.videosRedundancy).to.have.lengthOf(1)
 121
 122   const stat = data.videosRedundancy[0]
 123   expect(stat.strategy).to.equal(strategy)
 124   expect(stat.totalSize).to.equal(102400)
 125   expect(stat.totalUsed).to.equal(0)
 126   expect(stat.totalVideoFiles).to.equal(0)
 127   expect(stat.totalVideos).to.equal(0)
 128 }
 129
 130 async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
 131   if (!videoUUID) videoUUID = video1Server2UUID
 132
 133   const webseeds = [
 134     'http://localhost:9001/static/webseed/' + videoUUID,
 135     'http://localhost:9002/static/webseed/' + videoUUID
 136   ]
 137
 138   for (const server of servers) {
 139     {
 140       const res = await getVideo(server.url, videoUUID)
 141
 142       const video: VideoDetails = res.body
 143
 144       for (const file of video.files) {
 145         checkMagnetWebseeds(file, webseeds, server)
 146
 147         // Only servers 1 and 2 have the video
 148         if (server.serverNumber !== 3) {
 149           await makeGetRequest({
 150             url: server.url,
 151             statusCodeExpected: 200,
 152             path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
 153             contentType: null
 154           })
 155         }
 156       }
 157     }
 158   }
 159
 160   for (const directory of [ 'test1', 'test2' ]) {
 161     const files = await readdir(join(root(), directory, 'videos'))
 162     expect(files).to.have.length.at.least(4)
 163
 164     for (const resolution of [ 240, 360, 480, 720 ]) {
 165       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 166     }
 167   }
 168 }
 169
 170 async function enableRedundancyOnServer1 () {
 171   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 172
 173   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 174   const follows: ActorFollow[] = res.body.data
 175   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 176   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 177
 178   expect(server3).to.not.be.undefined
 179   expect(server3.following.hostRedundancyAllowed).to.be.false
 180
 181   expect(server2).to.not.be.undefined
 182   expect(server2.following.hostRedundancyAllowed).to.be.true
 183 }
 184
 185 async function cleanServers () {
 186   killallServers(servers)
 187 }
 188
 189 describe('Test videos redundancy', function () {
 190
 191   describe('With most-views strategy', function () {
 192     const strategy = 'most-views'
 193
 194     before(function () {
 195       this.timeout(120000)
 196
 197       return runServers(strategy)
 198     })
 199
 200     it('Should have 1 webseed on the first video', async function () {
 201       await check1WebSeed(strategy)
 202       await checkStatsWith1Webseed(strategy)
 203     })
 204
 205     it('Should enable redundancy on server 1', function () {
 206       return enableRedundancyOnServer1()
 207     })
 208
 209     it('Should have 2 webseed on the first video', async function () {
 210       this.timeout(40000)
 211
 212       await waitJobs(servers)
 213       await wait(15000)
 214       await waitJobs(servers)
 215
 216       await check2Webseeds(strategy)
 217       await checkStatsWith2Webseed(strategy)
 218     })
 219
 220     after(function () {
 221       return cleanServers()
 222     })
 223   })
 224
 225   describe('With trending strategy', function () {
 226     const strategy = 'trending'
 227
 228     before(function () {
 229       this.timeout(120000)
 230
 231       return runServers(strategy)
 232     })
 233
 234     it('Should have 1 webseed on the first video', async function () {
 235       await check1WebSeed(strategy)
 236       await checkStatsWith1Webseed(strategy)
 237     })
 238
 239     it('Should enable redundancy on server 1', function () {
 240       return enableRedundancyOnServer1()
 241     })
 242
 243     it('Should have 2 webseed on the first video', async function () {
 244       this.timeout(40000)
 245
 246       await waitJobs(servers)
 247       await wait(15000)
 248       await waitJobs(servers)
 249
 250       await check2Webseeds(strategy)
 251       await checkStatsWith2Webseed(strategy)
 252     })
 253
 254     after(function () {
 255       return cleanServers()
 256     })
 257   })
 258
 259   describe('With recently added strategy', function () {
 260     const strategy = 'recently-added'
 261
 262     before(function () {
 263       this.timeout(120000)
 264
 265       return runServers(strategy, { min_views: 3 })
 266     })
 267
 268     it('Should have 1 webseed on the first video', async function () {
 269       await check1WebSeed(strategy)
 270       await checkStatsWith1Webseed(strategy)
 271     })
 272
 273     it('Should enable redundancy on server 1', function () {
 274       return enableRedundancyOnServer1()
 275     })
 276
 277     it('Should still have 1 webseed on the first video', async function () {
 278       this.timeout(40000)
 279
 280       await waitJobs(servers)
 281       await wait(15000)
 282       await waitJobs(servers)
 283
 284       await check1WebSeed(strategy)
 285       await checkStatsWith1Webseed(strategy)
 286     })
 287
 288     it('Should view 2 times the first video to have > min_views config', async function () {
 289       this.timeout(40000)
 290
 291       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 292       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 293
 294       await wait(10000)
 295       await waitJobs(servers)
 296     })
 297
 298     it('Should have 2 webseed on the first video', async function () {
 299       this.timeout(40000)
 300
 301       await waitJobs(servers)
 302       await wait(15000)
 303       await waitJobs(servers)
 304
 305       await check2Webseeds(strategy)
 306       await checkStatsWith2Webseed(strategy)
 307     })
 308
 309     after(function () {
 310       return cleanServers()
 311     })
 312   })
 313
 314   describe('Test expiration', function () {
 315     const strategy = 'recently-added'
 316
 317     async function checkContains (servers: ServerInfo[], str: string) {
 318       for (const server of servers) {
 319         const res = await getVideo(server.url, video1Server2UUID)
 320         const video: VideoDetails = res.body
 321
 322         for (const f of video.files) {
 323           expect(f.magnetUri).to.contain(str)
 324         }
 325       }
 326     }
 327
 328     async function checkNotContains (servers: ServerInfo[], str: string) {
 329       for (const server of servers) {
 330         const res = await getVideo(server.url, video1Server2UUID)
 331         const video: VideoDetails = res.body
 332
 333         for (const f of video.files) {
 334           expect(f.magnetUri).to.not.contain(str)
 335         }
 336       }
 337     }
 338
 339     before(async function () {
 340       this.timeout(120000)
 341
 342       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 343
 344       await enableRedundancyOnServer1()
 345     })
 346
 347     it('Should still have 2 webseeds after 10 seconds', async function () {
 348       this.timeout(40000)
 349
 350       await wait(10000)
 351
 352       try {
 353         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 354       } catch {
 355         // Maybe a server deleted a redundancy in the scheduler
 356         await wait(2000)
 357
 358         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 359       }
 360     })
 361
 362     it('Should stop server 1 and expire video redundancy', async function () {
 363       this.timeout(40000)
 364
 365       killallServers([ servers[0] ])
 366
 367       await wait(10000)
 368
 369       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 370     })
 371
 372     after(function () {
 373       return killallServers([ servers[1], servers[2] ])
 374     })
 375   })
 376
 377   describe('Test file replacement', function () {
 378     let video2Server2UUID: string
 379     const strategy = 'recently-added'
 380
 381     before(async function () {
 382       this.timeout(120000)
 383
 384       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 385
 386       await enableRedundancyOnServer1()
 387
 388       await waitJobs(servers)
 389       await wait(5000)
 390       await waitJobs(servers)
 391
 392       await check2Webseeds(strategy)
 393       await checkStatsWith2Webseed(strategy)
 394
 395       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 396       video2Server2UUID = res.body.video.uuid
 397     })
 398
 399     it('Should cache video 2 webseed on the first video', async function () {
 400       this.timeout(40000)
 401       this.retries(3)
 402
 403       await waitJobs(servers)
 404
 405       await wait(7000)
 406
 407       await check1WebSeed(strategy, video1Server2UUID)
 408       await check2Webseeds(strategy, video2Server2UUID)
 409     })
 410
 411     after(function () {
 412       return cleanServers()
 413     })
 414   })
 415 })