server/tests/api/server/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   doubleFollow,
   8   flushAndRunMultipleServers,
   9   getFollowingListPaginationAndSort,
  10   getVideo,
  11   immutableAssign,
  12   killallServers, makeGetRequest,
  13   root,
  14   ServerInfo,
  15   setAccessTokensToServers,
  16   uploadVideo,
  17   viewVideo,
  18   wait
  19 } from '../../utils'
  20 import { waitJobs } from '../../utils/server/jobs'
  21 import * as magnetUtil from 'magnet-uri'
  22 import { updateRedundancy } from '../../utils/server/redundancy'
  23 import { ActorFollow } from '../../../../shared/models/actors'
  24 import { readdir } from 'fs-extra'
  25 import { join } from 'path'
  26 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  27 import { getStats } from '../../utils/server/stats'
  28 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  29
  30 const expect = chai.expect
  31
  32 let servers: ServerInfo[] = []
  33 let video1Server2UUID: string
  34
  35 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  36   const parsed = magnetUtil.decode(file.magnetUri)
  37
  38   for (const ws of baseWebseeds) {
  39     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  40     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  41   }
  42 }
  43
  44 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  45   const config = {
  46     redundancy: {
  47       videos: {
  48         check_interval: '5 seconds',
  49         strategies: [
  50           immutableAssign({
  51             min_lifetime: '1 hour',
  52             strategy: strategy,
  53             size: '100KB'
  54           }, additionalParams)
  55         ]
  56       }
  57     }
  58   }
  59   servers = await flushAndRunMultipleServers(3, config)
  60
  61   // Get the access tokens
  62   await setAccessTokensToServers(servers)
  63
  64   {
  65     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  66     video1Server2UUID = res.body.video.uuid
  67
  68     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  69   }
  70
  71   await waitJobs(servers)
  72
  73   // Server 1 and server 2 follow each other
  74   await doubleFollow(servers[ 0 ], servers[ 1 ])
  75   // Server 1 and server 3 follow each other
  76   await doubleFollow(servers[ 0 ], servers[ 2 ])
  77   // Server 2 and server 3 follow each other
  78   await doubleFollow(servers[ 1 ], servers[ 2 ])
  79
  80   await waitJobs(servers)
  81 }
  82
  83 async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
  84   if (!videoUUID) videoUUID = video1Server2UUID
  85
  86   const webseeds = [
  87     'http://localhost:9002/static/webseed/' + videoUUID
  88   ]
  89
  90   for (const server of servers) {
  91     {
  92       const res = await getVideo(server.url, videoUUID)
  93
  94       const video: VideoDetails = res.body
  95       for (const f of video.files) {
  96         checkMagnetWebseeds(f, webseeds, server)
  97       }
  98     }
  99   }
 100 }
 101
 102 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 103   const res = await getStats(servers[0].url)
 104   const data: ServerStats = res.body
 105
 106   expect(data.videosRedundancy).to.have.lengthOf(1)
 107   const stat = data.videosRedundancy[0]
 108
 109   expect(stat.strategy).to.equal(strategy)
 110   expect(stat.totalSize).to.equal(102400)
 111   expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
 112   expect(stat.totalVideoFiles).to.equal(4)
 113   expect(stat.totalVideos).to.equal(1)
 114 }
 115
 116 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 117   const res = await getStats(servers[0].url)
 118   const data: ServerStats = res.body
 119
 120   expect(data.videosRedundancy).to.have.lengthOf(1)
 121
 122   const stat = data.videosRedundancy[0]
 123   expect(stat.strategy).to.equal(strategy)
 124   expect(stat.totalSize).to.equal(102400)
 125   expect(stat.totalUsed).to.equal(0)
 126   expect(stat.totalVideoFiles).to.equal(0)
 127   expect(stat.totalVideos).to.equal(0)
 128 }
 129
 130 async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
 131   if (!videoUUID) videoUUID = video1Server2UUID
 132
 133   const webseeds = [
 134     'http://localhost:9001/static/webseed/' + videoUUID,
 135     'http://localhost:9002/static/webseed/' + videoUUID
 136   ]
 137
 138   for (const server of servers) {
 139     {
 140       const res = await getVideo(server.url, videoUUID)
 141
 142       const video: VideoDetails = res.body
 143
 144       for (const file of video.files) {
 145         checkMagnetWebseeds(file, webseeds, server)
 146       }
 147     }
 148   }
 149
 150   for (const url of [ 'http://localhost:9001', 'http://localhost:9002' ]) {
 151     await makeGetRequest({
 152       url,
 153       statusCodeExpected: 200,
 154       path: '/static/webseed/' + videoUUID,
 155       contentType: null
 156     })
 157   }
 158
 159   for (const directory of [ 'test1', 'test2' ]) {
 160     const files = await readdir(join(root(), directory, 'videos'))
 161     expect(files).to.have.length.at.least(4)
 162
 163     for (const resolution of [ 240, 360, 480, 720 ]) {
 164       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 165     }
 166   }
 167 }
 168
 169 async function enableRedundancyOnServer1 () {
 170   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 171
 172   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 173   const follows: ActorFollow[] = res.body.data
 174   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 175   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 176
 177   expect(server3).to.not.be.undefined
 178   expect(server3.following.hostRedundancyAllowed).to.be.false
 179
 180   expect(server2).to.not.be.undefined
 181   expect(server2.following.hostRedundancyAllowed).to.be.true
 182 }
 183
 184 async function cleanServers () {
 185   killallServers(servers)
 186 }
 187
 188 describe('Test videos redundancy', function () {
 189
 190   describe('With most-views strategy', function () {
 191     const strategy = 'most-views'
 192
 193     before(function () {
 194       this.timeout(120000)
 195
 196       return runServers(strategy)
 197     })
 198
 199     it('Should have 1 webseed on the first video', async function () {
 200       await check1WebSeed(strategy)
 201       await checkStatsWith1Webseed(strategy)
 202     })
 203
 204     it('Should enable redundancy on server 1', function () {
 205       return enableRedundancyOnServer1()
 206     })
 207
 208     it('Should have 2 webseed on the first video', async function () {
 209       this.timeout(40000)
 210
 211       await waitJobs(servers)
 212       await wait(15000)
 213       await waitJobs(servers)
 214
 215       await check2Webseeds(strategy)
 216       await checkStatsWith2Webseed(strategy)
 217     })
 218
 219     after(function () {
 220       return cleanServers()
 221     })
 222   })
 223
 224   describe('With trending strategy', function () {
 225     const strategy = 'trending'
 226
 227     before(function () {
 228       this.timeout(120000)
 229
 230       return runServers(strategy)
 231     })
 232
 233     it('Should have 1 webseed on the first video', async function () {
 234       await check1WebSeed(strategy)
 235       await checkStatsWith1Webseed(strategy)
 236     })
 237
 238     it('Should enable redundancy on server 1', function () {
 239       return enableRedundancyOnServer1()
 240     })
 241
 242     it('Should have 2 webseed on the first video', async function () {
 243       this.timeout(40000)
 244
 245       await waitJobs(servers)
 246       await wait(15000)
 247       await waitJobs(servers)
 248
 249       await check2Webseeds(strategy)
 250       await checkStatsWith2Webseed(strategy)
 251     })
 252
 253     after(function () {
 254       return cleanServers()
 255     })
 256   })
 257
 258   describe('With recently added strategy', function () {
 259     const strategy = 'recently-added'
 260
 261     before(function () {
 262       this.timeout(120000)
 263
 264       return runServers(strategy, { min_views: 3 })
 265     })
 266
 267     it('Should have 1 webseed on the first video', async function () {
 268       await check1WebSeed(strategy)
 269       await checkStatsWith1Webseed(strategy)
 270     })
 271
 272     it('Should enable redundancy on server 1', function () {
 273       return enableRedundancyOnServer1()
 274     })
 275
 276     it('Should still have 1 webseed on the first video', async function () {
 277       this.timeout(40000)
 278
 279       await waitJobs(servers)
 280       await wait(15000)
 281       await waitJobs(servers)
 282
 283       await check1WebSeed(strategy)
 284       await checkStatsWith1Webseed(strategy)
 285     })
 286
 287     it('Should view 2 times the first video to have > min_views config', async function () {
 288       this.timeout(40000)
 289
 290       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 291       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 292
 293       await wait(10000)
 294       await waitJobs(servers)
 295     })
 296
 297     it('Should have 2 webseed on the first video', async function () {
 298       this.timeout(40000)
 299
 300       await waitJobs(servers)
 301       await wait(15000)
 302       await waitJobs(servers)
 303
 304       await check2Webseeds(strategy)
 305       await checkStatsWith2Webseed(strategy)
 306     })
 307
 308     after(function () {
 309       return cleanServers()
 310     })
 311   })
 312
 313   describe('Test expiration', function () {
 314     const strategy = 'recently-added'
 315
 316     async function checkContains (servers: ServerInfo[], str: string) {
 317       for (const server of servers) {
 318         const res = await getVideo(server.url, video1Server2UUID)
 319         const video: VideoDetails = res.body
 320
 321         for (const f of video.files) {
 322           expect(f.magnetUri).to.contain(str)
 323         }
 324       }
 325     }
 326
 327     async function checkNotContains (servers: ServerInfo[], str: string) {
 328       for (const server of servers) {
 329         const res = await getVideo(server.url, video1Server2UUID)
 330         const video: VideoDetails = res.body
 331
 332         for (const f of video.files) {
 333           expect(f.magnetUri).to.not.contain(str)
 334         }
 335       }
 336     }
 337
 338     before(async function () {
 339       this.timeout(120000)
 340
 341       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 342
 343       await enableRedundancyOnServer1()
 344     })
 345
 346     it('Should still have 2 webseeds after 10 seconds', async function () {
 347       this.timeout(40000)
 348
 349       await wait(10000)
 350
 351       try {
 352         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 353       } catch {
 354         // Maybe a server deleted a redundancy in the scheduler
 355         await wait(2000)
 356
 357         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 358       }
 359     })
 360
 361     it('Should stop server 1 and expire video redundancy', async function () {
 362       this.timeout(40000)
 363
 364       killallServers([ servers[0] ])
 365
 366       await wait(10000)
 367
 368       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 369     })
 370
 371     after(function () {
 372       return killallServers([ servers[1], servers[2] ])
 373     })
 374   })
 375
 376   describe('Test file replacement', function () {
 377     let video2Server2UUID: string
 378     const strategy = 'recently-added'
 379
 380     before(async function () {
 381       this.timeout(120000)
 382
 383       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 384
 385       await enableRedundancyOnServer1()
 386
 387       await waitJobs(servers)
 388       await wait(5000)
 389       await waitJobs(servers)
 390
 391       await check2Webseeds(strategy)
 392       await checkStatsWith2Webseed(strategy)
 393
 394       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 395       video2Server2UUID = res.body.video.uuid
 396     })
 397
 398     it('Should cache video 2 webseed on the first video', async function () {
 399       this.timeout(40000)
 400       this.retries(3)
 401
 402       await waitJobs(servers)
 403
 404       await wait(7000)
 405
 406       await check1WebSeed(strategy, video1Server2UUID)
 407       await check2Webseeds(strategy, video2Server2UUID)
 408     })
 409
 410     after(function () {
 411       return cleanServers()
 412     })
 413   })
 414 })