server/tests/api/server/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   doubleFollow,
   8   flushAndRunMultipleServers,
   9   getFollowingListPaginationAndSort,
  10   getVideo,
  11   immutableAssign,
  12   killallServers,
  13   root,
  14   ServerInfo,
  15   setAccessTokensToServers,
  16   uploadVideo,
  17   viewVideo,
  18   wait
  19 } from '../../utils'
  20 import { waitJobs } from '../../utils/server/jobs'
  21 import * as magnetUtil from 'magnet-uri'
  22 import { updateRedundancy } from '../../utils/server/redundancy'
  23 import { ActorFollow } from '../../../../shared/models/actors'
  24 import { readdir } from 'fs-extra'
  25 import { join } from 'path'
  26 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  27 import { getStats } from '../../utils/server/stats'
  28 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  29
  30 const expect = chai.expect
  31
  32 let servers: ServerInfo[] = []
  33 let video1Server2UUID: string
  34
  35 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  36   const parsed = magnetUtil.decode(file.magnetUri)
  37
  38   for (const ws of baseWebseeds) {
  39     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  40     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  41   }
  42 }
  43
  44 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  45   const config = {
  46     redundancy: {
  47       videos: {
  48         check_interval: '5 seconds',
  49         strategies: [
  50           immutableAssign({
  51             min_lifetime: '1 hour',
  52             strategy: strategy,
  53             size: '100KB'
  54           }, additionalParams)
  55         ]
  56       }
  57     }
  58   }
  59   servers = await flushAndRunMultipleServers(3, config)
  60
  61   // Get the access tokens
  62   await setAccessTokensToServers(servers)
  63
  64   {
  65     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  66     video1Server2UUID = res.body.video.uuid
  67
  68     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  69   }
  70
  71   await waitJobs(servers)
  72
  73   // Server 1 and server 2 follow each other
  74   await doubleFollow(servers[ 0 ], servers[ 1 ])
  75   // Server 1 and server 3 follow each other
  76   await doubleFollow(servers[ 0 ], servers[ 2 ])
  77   // Server 2 and server 3 follow each other
  78   await doubleFollow(servers[ 1 ], servers[ 2 ])
  79
  80   await waitJobs(servers)
  81 }
  82
  83 async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
  84   if (!videoUUID) videoUUID = video1Server2UUID
  85
  86   const webseeds = [
  87     'http://localhost:9002/static/webseed/' + videoUUID
  88   ]
  89
  90   for (const server of servers) {
  91     {
  92       const res = await getVideo(server.url, videoUUID)
  93
  94       const video: VideoDetails = res.body
  95       for (const f of video.files) {
  96         checkMagnetWebseeds(f, webseeds, server)
  97       }
  98     }
  99   }
 100 }
 101
 102 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 103   const res = await getStats(servers[0].url)
 104   const data: ServerStats = res.body
 105
 106   expect(data.videosRedundancy).to.have.lengthOf(1)
 107   const stat = data.videosRedundancy[0]
 108
 109   expect(stat.strategy).to.equal(strategy)
 110   expect(stat.totalSize).to.equal(102400)
 111   expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
 112   expect(stat.totalVideoFiles).to.equal(4)
 113   expect(stat.totalVideos).to.equal(1)
 114 }
 115
 116 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 117   const res = await getStats(servers[0].url)
 118   const data: ServerStats = res.body
 119
 120   expect(data.videosRedundancy).to.have.lengthOf(1)
 121
 122   const stat = data.videosRedundancy[0]
 123   expect(stat.strategy).to.equal(strategy)
 124   expect(stat.totalSize).to.equal(102400)
 125   expect(stat.totalUsed).to.equal(0)
 126   expect(stat.totalVideoFiles).to.equal(0)
 127   expect(stat.totalVideos).to.equal(0)
 128 }
 129
 130 async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
 131   if (!videoUUID) videoUUID = video1Server2UUID
 132
 133   const webseeds = [
 134     'http://localhost:9001/static/webseed/' + videoUUID,
 135     'http://localhost:9002/static/webseed/' + videoUUID
 136   ]
 137
 138   for (const server of servers) {
 139     {
 140       const res = await getVideo(server.url, videoUUID)
 141
 142       const video: VideoDetails = res.body
 143
 144       for (const file of video.files) {
 145         checkMagnetWebseeds(file, webseeds, server)
 146       }
 147     }
 148   }
 149
 150   const files = await readdir(join(root(), 'test1', 'videos'))
 151   expect(files).to.have.lengthOf(4)
 152
 153   for (const resolution of [ 240, 360, 480, 720 ]) {
 154     expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 155   }
 156 }
 157
 158 async function enableRedundancyOnServer1 () {
 159   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 160
 161   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 162   const follows: ActorFollow[] = res.body.data
 163   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 164   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 165
 166   expect(server3).to.not.be.undefined
 167   expect(server3.following.hostRedundancyAllowed).to.be.false
 168
 169   expect(server2).to.not.be.undefined
 170   expect(server2.following.hostRedundancyAllowed).to.be.true
 171 }
 172
 173 async function cleanServers () {
 174   killallServers(servers)
 175 }
 176
 177 describe('Test videos redundancy', function () {
 178
 179   describe('With most-views strategy', function () {
 180     const strategy = 'most-views'
 181
 182     before(function () {
 183       this.timeout(120000)
 184
 185       return runServers(strategy)
 186     })
 187
 188     it('Should have 1 webseed on the first video', async function () {
 189       await check1WebSeed(strategy)
 190       await checkStatsWith1Webseed(strategy)
 191     })
 192
 193     it('Should enable redundancy on server 1', function () {
 194       return enableRedundancyOnServer1()
 195     })
 196
 197     it('Should have 2 webseed on the first video', async function () {
 198       this.timeout(40000)
 199
 200       await waitJobs(servers)
 201       await wait(15000)
 202       await waitJobs(servers)
 203
 204       await check2Webseeds(strategy)
 205       await checkStatsWith2Webseed(strategy)
 206     })
 207
 208     after(function () {
 209       return cleanServers()
 210     })
 211   })
 212
 213   describe('With trending strategy', function () {
 214     const strategy = 'trending'
 215
 216     before(function () {
 217       this.timeout(120000)
 218
 219       return runServers(strategy)
 220     })
 221
 222     it('Should have 1 webseed on the first video', async function () {
 223       await check1WebSeed(strategy)
 224       await checkStatsWith1Webseed(strategy)
 225     })
 226
 227     it('Should enable redundancy on server 1', function () {
 228       return enableRedundancyOnServer1()
 229     })
 230
 231     it('Should have 2 webseed on the first video', async function () {
 232       this.timeout(40000)
 233
 234       await waitJobs(servers)
 235       await wait(15000)
 236       await waitJobs(servers)
 237
 238       await check2Webseeds(strategy)
 239       await checkStatsWith2Webseed(strategy)
 240     })
 241
 242     after(function () {
 243       return cleanServers()
 244     })
 245   })
 246
 247   describe('With recently added strategy', function () {
 248     const strategy = 'recently-added'
 249
 250     before(function () {
 251       this.timeout(120000)
 252
 253       return runServers(strategy, { min_views: 3 })
 254     })
 255
 256     it('Should have 1 webseed on the first video', async function () {
 257       await check1WebSeed(strategy)
 258       await checkStatsWith1Webseed(strategy)
 259     })
 260
 261     it('Should enable redundancy on server 1', function () {
 262       return enableRedundancyOnServer1()
 263     })
 264
 265     it('Should still have 1 webseed on the first video', async function () {
 266       this.timeout(40000)
 267
 268       await waitJobs(servers)
 269       await wait(15000)
 270       await waitJobs(servers)
 271
 272       await check1WebSeed(strategy)
 273       await checkStatsWith1Webseed(strategy)
 274     })
 275
 276     it('Should view 2 times the first video to have > min_views config', async function () {
 277       this.timeout(40000)
 278
 279       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 280       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 281
 282       await wait(10000)
 283       await waitJobs(servers)
 284     })
 285
 286     it('Should have 2 webseed on the first video', async function () {
 287       this.timeout(40000)
 288
 289       await waitJobs(servers)
 290       await wait(15000)
 291       await waitJobs(servers)
 292
 293       await check2Webseeds(strategy)
 294       await checkStatsWith2Webseed(strategy)
 295     })
 296
 297     after(function () {
 298       return cleanServers()
 299     })
 300   })
 301
 302   describe('Test expiration', function () {
 303     const strategy = 'recently-added'
 304
 305     async function checkContains (servers: ServerInfo[], str: string) {
 306       for (const server of servers) {
 307         const res = await getVideo(server.url, video1Server2UUID)
 308         const video: VideoDetails = res.body
 309
 310         for (const f of video.files) {
 311           expect(f.magnetUri).to.contain(str)
 312         }
 313       }
 314     }
 315
 316     async function checkNotContains (servers: ServerInfo[], str: string) {
 317       for (const server of servers) {
 318         const res = await getVideo(server.url, video1Server2UUID)
 319         const video: VideoDetails = res.body
 320
 321         for (const f of video.files) {
 322           expect(f.magnetUri).to.not.contain(str)
 323         }
 324       }
 325     }
 326
 327     before(async function () {
 328       this.timeout(120000)
 329
 330       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 331
 332       await enableRedundancyOnServer1()
 333     })
 334
 335     it('Should still have 2 webseeds after 10 seconds', async function () {
 336       this.timeout(40000)
 337
 338       await wait(10000)
 339
 340       try {
 341         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 342       } catch {
 343         // Maybe a server deleted a redundancy in the scheduler
 344         await wait(2000)
 345
 346         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 347       }
 348     })
 349
 350     it('Should stop server 1 and expire video redundancy', async function () {
 351       this.timeout(40000)
 352
 353       killallServers([ servers[0] ])
 354
 355       await wait(10000)
 356
 357       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 358     })
 359
 360     after(function () {
 361       return killallServers([ servers[1], servers[2] ])
 362     })
 363   })
 364
 365   describe('Test file replacement', function () {
 366     let video2Server2UUID: string
 367     const strategy = 'recently-added'
 368
 369     before(async function () {
 370       this.timeout(120000)
 371
 372       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 373
 374       await enableRedundancyOnServer1()
 375
 376       await waitJobs(servers)
 377       await wait(5000)
 378       await waitJobs(servers)
 379
 380       await check2Webseeds(strategy)
 381       await checkStatsWith2Webseed(strategy)
 382
 383       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 384       video2Server2UUID = res.body.video.uuid
 385     })
 386
 387     it('Should cache video 2 webseed on the first video', async function () {
 388       this.timeout(40000)
 389       this.retries(3)
 390
 391       await waitJobs(servers)
 392
 393       await wait(7000)
 394
 395       await check1WebSeed(strategy, video1Server2UUID)
 396       await check2Webseeds(strategy, video2Server2UUID)
 397     })
 398
 399     after(function () {
 400       return cleanServers()
 401     })
 402   })
 403 })