server/tests/api/redundancy/redundancy.ts

   1 /* tslint:disable:no-unused-expression */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { VideoDetails } from '../../../../shared/models/videos'
   6 import {
   7   doubleFollow,
   8   flushAndRunMultipleServers,
   9   getFollowingListPaginationAndSort,
  10   getVideo,
  11   immutableAssign,
  12   killallServers, makeGetRequest,
  13   root,
  14   ServerInfo,
  15   setAccessTokensToServers, unfollow,
  16   uploadVideo,
  17   viewVideo,
  18   wait,
  19   waitUntilLog,
  20   checkVideoFilesWereRemoved, removeVideo, getVideoWithToken
  21 } from '../../utils'
  22 import { waitJobs } from '../../utils/server/jobs'
  23 import * as magnetUtil from 'magnet-uri'
  24 import { updateRedundancy } from '../../utils/server/redundancy'
  25 import { ActorFollow } from '../../../../shared/models/actors'
  26 import { readdir } from 'fs-extra'
  27 import { join } from 'path'
  28 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
  29 import { getStats } from '../../utils/server/stats'
  30 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
  31
  32 const expect = chai.expect
  33
  34 let servers: ServerInfo[] = []
  35 let video1Server2UUID: string
  36
  37 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
  38   const parsed = magnetUtil.decode(file.magnetUri)
  39
  40   for (const ws of baseWebseeds) {
  41     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
  42     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
  43   }
  44
  45   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
  46 }
  47
  48 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
  49   const config = {
  50     redundancy: {
  51       videos: {
  52         check_interval: '5 seconds',
  53         strategies: [
  54           immutableAssign({
  55             min_lifetime: '1 hour',
  56             strategy: strategy,
  57             size: '200KB'
  58           }, additionalParams)
  59         ]
  60       }
  61     }
  62   }
  63   servers = await flushAndRunMultipleServers(3, config)
  64
  65   // Get the access tokens
  66   await setAccessTokensToServers(servers)
  67
  68   {
  69     const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
  70     video1Server2UUID = res.body.video.uuid
  71
  72     await viewVideo(servers[ 1 ].url, video1Server2UUID)
  73   }
  74
  75   await waitJobs(servers)
  76
  77   // Server 1 and server 2 follow each other
  78   await doubleFollow(servers[ 0 ], servers[ 1 ])
  79   // Server 1 and server 3 follow each other
  80   await doubleFollow(servers[ 0 ], servers[ 2 ])
  81   // Server 2 and server 3 follow each other
  82   await doubleFollow(servers[ 1 ], servers[ 2 ])
  83
  84   await waitJobs(servers)
  85 }
  86
  87 async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
  88   if (!videoUUID) videoUUID = video1Server2UUID
  89
  90   const webseeds = [
  91     'http://localhost:9002/static/webseed/' + videoUUID
  92   ]
  93
  94   for (const server of servers) {
  95     {
  96       // With token to avoid issues with video follow constraints
  97       const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
  98
  99       const video: VideoDetails = res.body
 100       for (const f of video.files) {
 101         checkMagnetWebseeds(f, webseeds, server)
 102       }
 103     }
 104   }
 105 }
 106
 107 async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
 108   const res = await getStats(servers[0].url)
 109   const data: ServerStats = res.body
 110
 111   expect(data.videosRedundancy).to.have.lengthOf(1)
 112   const stat = data.videosRedundancy[0]
 113
 114   expect(stat.strategy).to.equal(strategy)
 115   expect(stat.totalSize).to.equal(204800)
 116   expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
 117   expect(stat.totalVideoFiles).to.equal(4)
 118   expect(stat.totalVideos).to.equal(1)
 119 }
 120
 121 async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
 122   const res = await getStats(servers[0].url)
 123   const data: ServerStats = res.body
 124
 125   expect(data.videosRedundancy).to.have.lengthOf(1)
 126
 127   const stat = data.videosRedundancy[0]
 128   expect(stat.strategy).to.equal(strategy)
 129   expect(stat.totalSize).to.equal(204800)
 130   expect(stat.totalUsed).to.equal(0)
 131   expect(stat.totalVideoFiles).to.equal(0)
 132   expect(stat.totalVideos).to.equal(0)
 133 }
 134
 135 async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
 136   if (!videoUUID) videoUUID = video1Server2UUID
 137
 138   const webseeds = [
 139     'http://localhost:9001/static/webseed/' + videoUUID,
 140     'http://localhost:9002/static/webseed/' + videoUUID
 141   ]
 142
 143   for (const server of servers) {
 144     const res = await getVideo(server.url, videoUUID)
 145
 146     const video: VideoDetails = res.body
 147
 148     for (const file of video.files) {
 149       checkMagnetWebseeds(file, webseeds, server)
 150
 151       // Only servers 1 and 2 have the video
 152       if (server.serverNumber !== 3) {
 153         await makeGetRequest({
 154           url: server.url,
 155           statusCodeExpected: 200,
 156           path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
 157           contentType: null
 158         })
 159       }
 160     }
 161   }
 162
 163   for (const directory of [ 'test1', 'test2' ]) {
 164     const files = await readdir(join(root(), directory, 'videos'))
 165     expect(files).to.have.length.at.least(4)
 166
 167     for (const resolution of [ 240, 360, 480, 720 ]) {
 168       expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
 169     }
 170   }
 171 }
 172
 173 async function enableRedundancyOnServer1 () {
 174   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 175
 176   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 177   const follows: ActorFollow[] = res.body.data
 178   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 179   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 180
 181   expect(server3).to.not.be.undefined
 182   expect(server3.following.hostRedundancyAllowed).to.be.false
 183
 184   expect(server2).to.not.be.undefined
 185   expect(server2.following.hostRedundancyAllowed).to.be.true
 186 }
 187
 188 async function disableRedundancyOnServer1 () {
 189   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
 190
 191   const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
 192   const follows: ActorFollow[] = res.body.data
 193   const server2 = follows.find(f => f.following.host === 'localhost:9002')
 194   const server3 = follows.find(f => f.following.host === 'localhost:9003')
 195
 196   expect(server3).to.not.be.undefined
 197   expect(server3.following.hostRedundancyAllowed).to.be.false
 198
 199   expect(server2).to.not.be.undefined
 200   expect(server2.following.hostRedundancyAllowed).to.be.false
 201 }
 202
 203 async function cleanServers () {
 204   killallServers(servers)
 205 }
 206
 207 describe('Test videos redundancy', function () {
 208
 209   describe('With most-views strategy', function () {
 210     const strategy = 'most-views'
 211
 212     before(function () {
 213       this.timeout(120000)
 214
 215       return runServers(strategy)
 216     })
 217
 218     it('Should have 1 webseed on the first video', async function () {
 219       await check1WebSeed(strategy)
 220       await checkStatsWith1Webseed(strategy)
 221     })
 222
 223     it('Should enable redundancy on server 1', function () {
 224       return enableRedundancyOnServer1()
 225     })
 226
 227     it('Should have 2 webseeds on the first video', async function () {
 228       this.timeout(40000)
 229
 230       await waitJobs(servers)
 231       await waitUntilLog(servers[0], 'Duplicated ', 4)
 232       await waitJobs(servers)
 233
 234       await check2Webseeds(strategy)
 235       await checkStatsWith2Webseed(strategy)
 236     })
 237
 238     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
 239       this.timeout(40000)
 240
 241       await disableRedundancyOnServer1()
 242
 243       await waitJobs(servers)
 244       await wait(5000)
 245
 246       await check1WebSeed(strategy)
 247
 248       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 249     })
 250
 251     after(function () {
 252       return cleanServers()
 253     })
 254   })
 255
 256   describe('With trending strategy', function () {
 257     const strategy = 'trending'
 258
 259     before(function () {
 260       this.timeout(120000)
 261
 262       return runServers(strategy)
 263     })
 264
 265     it('Should have 1 webseed on the first video', async function () {
 266       await check1WebSeed(strategy)
 267       await checkStatsWith1Webseed(strategy)
 268     })
 269
 270     it('Should enable redundancy on server 1', function () {
 271       return enableRedundancyOnServer1()
 272     })
 273
 274     it('Should have 2 webseeds on the first video', async function () {
 275       this.timeout(40000)
 276
 277       await waitJobs(servers)
 278       await waitUntilLog(servers[0], 'Duplicated ', 4)
 279       await waitJobs(servers)
 280
 281       await check2Webseeds(strategy)
 282       await checkStatsWith2Webseed(strategy)
 283     })
 284
 285     it('Should unfollow on server 1 and remove duplicated videos', async function () {
 286       this.timeout(40000)
 287
 288       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 289
 290       await waitJobs(servers)
 291       await wait(5000)
 292
 293       await check1WebSeed(strategy)
 294
 295       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
 296     })
 297
 298     after(function () {
 299       return cleanServers()
 300     })
 301   })
 302
 303   describe('With recently added strategy', function () {
 304     const strategy = 'recently-added'
 305
 306     before(function () {
 307       this.timeout(120000)
 308
 309       return runServers(strategy, { min_views: 3 })
 310     })
 311
 312     it('Should have 1 webseed on the first video', async function () {
 313       await check1WebSeed(strategy)
 314       await checkStatsWith1Webseed(strategy)
 315     })
 316
 317     it('Should enable redundancy on server 1', function () {
 318       return enableRedundancyOnServer1()
 319     })
 320
 321     it('Should still have 1 webseed on the first video', async function () {
 322       this.timeout(40000)
 323
 324       await waitJobs(servers)
 325       await wait(15000)
 326       await waitJobs(servers)
 327
 328       await check1WebSeed(strategy)
 329       await checkStatsWith1Webseed(strategy)
 330     })
 331
 332     it('Should view 2 times the first video to have > min_views config', async function () {
 333       this.timeout(40000)
 334
 335       await viewVideo(servers[ 0 ].url, video1Server2UUID)
 336       await viewVideo(servers[ 2 ].url, video1Server2UUID)
 337
 338       await wait(10000)
 339       await waitJobs(servers)
 340     })
 341
 342     it('Should have 2 webseeds on the first video', async function () {
 343       this.timeout(40000)
 344
 345       await waitJobs(servers)
 346       await waitUntilLog(servers[0], 'Duplicated ', 4)
 347       await waitJobs(servers)
 348
 349       await check2Webseeds(strategy)
 350       await checkStatsWith2Webseed(strategy)
 351     })
 352
 353     it('Should remove the video and the redundancy files', async function () {
 354       this.timeout(20000)
 355
 356       await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
 357
 358       await waitJobs(servers)
 359
 360       for (const server of servers) {
 361         await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber)
 362       }
 363     })
 364
 365     after(function () {
 366       return cleanServers()
 367     })
 368   })
 369
 370   describe('Test expiration', function () {
 371     const strategy = 'recently-added'
 372
 373     async function checkContains (servers: ServerInfo[], str: string) {
 374       for (const server of servers) {
 375         const res = await getVideo(server.url, video1Server2UUID)
 376         const video: VideoDetails = res.body
 377
 378         for (const f of video.files) {
 379           expect(f.magnetUri).to.contain(str)
 380         }
 381       }
 382     }
 383
 384     async function checkNotContains (servers: ServerInfo[], str: string) {
 385       for (const server of servers) {
 386         const res = await getVideo(server.url, video1Server2UUID)
 387         const video: VideoDetails = res.body
 388
 389         for (const f of video.files) {
 390           expect(f.magnetUri).to.not.contain(str)
 391         }
 392       }
 393     }
 394
 395     before(async function () {
 396       this.timeout(120000)
 397
 398       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 399
 400       await enableRedundancyOnServer1()
 401     })
 402
 403     it('Should still have 2 webseeds after 10 seconds', async function () {
 404       this.timeout(40000)
 405
 406       await wait(10000)
 407
 408       try {
 409         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 410       } catch {
 411         // Maybe a server deleted a redundancy in the scheduler
 412         await wait(2000)
 413
 414         await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
 415       }
 416     })
 417
 418     it('Should stop server 1 and expire video redundancy', async function () {
 419       this.timeout(40000)
 420
 421       killallServers([ servers[0] ])
 422
 423       await wait(15000)
 424
 425       await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
 426     })
 427
 428     after(function () {
 429       return killallServers([ servers[1], servers[2] ])
 430     })
 431   })
 432
 433   describe('Test file replacement', function () {
 434     let video2Server2UUID: string
 435     const strategy = 'recently-added'
 436
 437     before(async function () {
 438       this.timeout(120000)
 439
 440       await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
 441
 442       await enableRedundancyOnServer1()
 443
 444       await waitJobs(servers)
 445       await waitUntilLog(servers[0], 'Duplicated ', 4)
 446       await waitJobs(servers)
 447
 448       await check2Webseeds(strategy)
 449       await checkStatsWith2Webseed(strategy)
 450
 451       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
 452       video2Server2UUID = res.body.video.uuid
 453     })
 454
 455     it('Should cache video 2 webseeds on the first video', async function () {
 456       this.timeout(120000)
 457
 458       await waitJobs(servers)
 459
 460       let checked = false
 461
 462       while (checked === false) {
 463         await wait(1000)
 464
 465         try {
 466           await check1WebSeed(strategy, video1Server2UUID)
 467           await check2Webseeds(strategy, video2Server2UUID)
 468
 469           checked = true
 470         } catch {
 471           checked = false
 472         }
 473       }
 474     })
 475
 476     after(function () {
 477       return cleanServers()
 478     })
 479   })
 480 })