server/tests/real-world/real-world.ts

   1 import * as program from 'commander'
   2
   3 // /!\ Before imports /!\
   4 process.env.NODE_ENV = 'test'
   5
   6 import { REQUESTS_INTERVAL } from '../../initializers/constants'
   7 import { Video, VideoRateType, VideoFile } from '../../../shared'
   8 import {
   9   ServerInfo as DefaultServerInfo,
  10   flushAndRunMultipleServers,
  11   setAccessTokensToServers,
  12   makeFriends,
  13   wait,
  14   killallServers,
  15   flushTests,
  16   uploadVideo,
  17   getVideosList,
  18   updateVideo,
  19   removeVideo,
  20   getVideo,
  21   getAllVideosListBy,
  22   getRequestsStats
  23 } from '../utils'
  24
  25 interface ServerInfo extends DefaultServerInfo {
  26   requestsNumber: number
  27 }
  28
  29 program
  30   .option('-c, --create [weight]', 'Weight for creating videos')
  31   .option('-r, --remove [weight]', 'Weight for removing videos')
  32   .option('-u, --update [weight]', 'Weight for updating videos')
  33   .option('-v, --view [weight]', 'Weight for viewing videos')
  34   .option('-l, --like [weight]', 'Weight for liking videos')
  35   .option('-s, --dislike [weight]', 'Weight for disliking videos')
  36   .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3)
  37   .option('-i, --interval-action [interval]', 'Interval in ms for an action')
  38   .option('-I, --interval-integrity [interval]', 'Interval in ms for an integrity check')
  39   .option('-f, --flush', 'Flush datas on exit')
  40   .option('-d, --difference', 'Display difference if integrity is not okay')
  41   .parse(process.argv)
  42
  43 const createWeight = program['create'] !== undefined ? parseInt(program['create'], 10) : 5
  44 const removeWeight = program['remove'] !== undefined ? parseInt(program['remove'], 10) : 4
  45 const updateWeight = program['update'] !== undefined ? parseInt(program['update'], 10) : 4
  46 const viewWeight = program['view'] !== undefined ? parseInt(program['view'], 10) : 4
  47 const likeWeight = program['like'] !== undefined ? parseInt(program['like'], 10) : 4
  48 const dislikeWeight = program['dislike'] !== undefined ? parseInt(program['dislike'], 10) : 4
  49 const flushAtExit = program['flush'] || false
  50 const actionInterval = program['intervalAction'] !== undefined ? parseInt(program['intervalAction'], 10) : 500
  51 const integrityInterval = program['intervalIntegrity'] !== undefined ? parseInt(program['intervalIntegrity'], 10) : 60000
  52 const displayDiffOnFail = program['difference'] || false
  53
  54 const numberOfPods = 6
  55
  56 console.log(
  57   'Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.',
  58   createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight
  59 )
  60
  61 if (flushAtExit) {
  62   console.log('Program will flush data on exit.')
  63 } else {
  64   console.log('Program will not flush data on exit.')
  65 }
  66 if (displayDiffOnFail) {
  67   console.log('Program will display diff on failure.')
  68 } else {
  69   console.log('Program will not display diff on failure')
  70 }
  71 console.log('Interval in ms for each action: %d.', actionInterval)
  72 console.log('Interval in ms for each integrity check: %d.', integrityInterval)
  73
  74 console.log('Run servers...')
  75
  76 start()
  77
  78 // ----------------------------------------------------------------------------
  79
  80 async function start () {
  81   const servers = await runServers(numberOfPods)
  82
  83   process.on('exit', async () => {
  84     await exitServers(servers, flushAtExit)
  85
  86     return
  87   })
  88   process.on('SIGINT', goodbye)
  89   process.on('SIGTERM', goodbye)
  90
  91   console.log('Servers ran')
  92   initializeRequestsPerServer(servers)
  93
  94   let checking = false
  95
  96   setInterval(async () => {
  97     if (checking === true) return
  98
  99     const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight)
 100
 101     const numServer = getRandomNumServer(servers)
 102     servers[numServer].requestsNumber++
 103
 104     if (rand < createWeight) {
 105       await upload(servers, numServer)
 106     } else if (rand < createWeight + updateWeight) {
 107       await update(servers, numServer)
 108     } else if (rand < createWeight + updateWeight + removeWeight) {
 109       await remove(servers, numServer)
 110     } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) {
 111       await view(servers, numServer)
 112     } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) {
 113       await like(servers, numServer)
 114     } else {
 115       await dislike(servers, numServer)
 116     }
 117   }, actionInterval)
 118
 119   // The function will check the consistency between servers (should have the same videos with same attributes...)
 120   setInterval(function () {
 121     if (checking === true) return
 122
 123     console.log('Checking integrity...')
 124     checking = true
 125
 126     const waitingInterval = setInterval(async () => {
 127       const pendingRequests = await isTherePendingRequests(servers)
 128       if (pendingRequests === true) {
 129         console.log('A server has pending requests, waiting...')
 130         return
 131       }
 132
 133       // Even if there are no pending request, wait some potential processes
 134       await wait(2000)
 135       await checkIntegrity(servers)
 136
 137       initializeRequestsPerServer(servers)
 138       checking = false
 139       clearInterval(waitingInterval)
 140     }, REQUESTS_INTERVAL)
 141   }, integrityInterval)
 142 }
 143
 144 function initializeRequestsPerServer (servers: ServerInfo[]) {
 145   servers.forEach(server => server.requestsNumber = 0)
 146 }
 147
 148 function getRandomInt (min, max) {
 149   return Math.floor(Math.random() * (max - min)) + min
 150 }
 151
 152 function getRandomNumServer (servers) {
 153   return getRandomInt(0, servers.length)
 154 }
 155
 156 async function runServers (numberOfPods: number) {
 157   const servers: ServerInfo[] = (await flushAndRunMultipleServers(numberOfPods))
 158     .map(s => Object.assign({ requestsNumber: 0 }, s))
 159
 160   // Get the access tokens
 161   await setAccessTokensToServers(servers)
 162
 163   await makeFriends(servers[1].url, servers[1].accessToken)
 164   await makeFriends(servers[0].url, servers[0].accessToken)
 165   await wait(1000)
 166
 167   await makeFriends(servers[3].url, servers[3].accessToken)
 168   await makeFriends(servers[5].url, servers[5].accessToken)
 169   await makeFriends(servers[4].url, servers[4].accessToken)
 170
 171   await wait(1000)
 172
 173   return servers
 174 }
 175
 176 async function exitServers (servers: ServerInfo[], flushAtExit: boolean) {
 177   killallServers(servers)
 178
 179   if (flushAtExit) await flushTests()
 180 }
 181
 182 function upload (servers: ServerInfo[], numServer: number) {
 183   console.log('Uploading video to server ' + numServer)
 184
 185   const videoAttributes = {
 186     name: Date.now() + ' name',
 187     category: 4,
 188     nsfw: false,
 189     licence: 2,
 190     language: 1,
 191     description: Date.now() + ' description',
 192     tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ],
 193     fixture: 'video_short1.webm'
 194   }
 195   return uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes)
 196 }
 197
 198 async function update (servers: ServerInfo[], numServer: number) {
 199   const res = await getVideosList(servers[numServer].url)
 200
 201   const videos = res.body.data.filter(video => video.isLocal === true)
 202   if (videos.length === 0) return undefined
 203
 204   const toUpdate = videos[getRandomInt(0, videos.length)].id
 205   const attributes = {
 206     name: Date.now() + ' name',
 207     description: Date.now() + ' description',
 208     tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ]
 209   }
 210
 211   console.log('Updating video of server ' + numServer)
 212
 213   return updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes)
 214 }
 215
 216 async function remove (servers: ServerInfo[], numServer: number) {
 217   const res = await getVideosList(servers[numServer].url)
 218   const videos = res.body.data.filter(video => video.isLocal === true)
 219   if (videos.length === 0) return undefined
 220
 221   const toRemove = videos[getRandomInt(0, videos.length)].id
 222
 223   console.log('Removing video from server ' + numServer)
 224   return removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove)
 225 }
 226
 227 async function view (servers: ServerInfo[], numServer: number) {
 228   const res = await getVideosList(servers[numServer].url)
 229
 230   const videos = res.body.data
 231   if (videos.length === 0) return undefined
 232
 233   const toView = videos[getRandomInt(0, videos.length)].id
 234
 235   console.log('Viewing video from server ' + numServer)
 236   return getVideo(servers[numServer].url, toView)
 237 }
 238
 239 function like (servers: ServerInfo[], numServer: number) {
 240   return rate(servers, numServer, 'like')
 241 }
 242
 243 function dislike (servers: ServerInfo[], numServer: number) {
 244   return rate(servers, numServer, 'dislike')
 245 }
 246
 247 async function rate (servers: ServerInfo[], numServer: number, rating: VideoRateType) {
 248   const res = await getVideosList(servers[numServer].url)
 249
 250   const videos = res.body.data
 251   if (videos.length === 0) return undefined
 252
 253   const toRate = videos[getRandomInt(0, videos.length)].id
 254
 255   console.log('Rating (%s) video from server %d', rating, numServer)
 256   return getVideo(servers[numServer].url, toRate)
 257 }
 258
 259 async function checkIntegrity (servers: ServerInfo[]) {
 260   const videos: Video[][] = []
 261   const tasks: Promise<any>[] = []
 262
 263   // Fetch all videos and remove some fields that can differ between pods
 264   for (const server of servers) {
 265     const p = getAllVideosListBy(server.url).then(res => videos.push(res.body.data))
 266     tasks.push(p)
 267   }
 268
 269   await Promise.all(tasks)
 270
 271   let i = 0
 272   for (const video of videos) {
 273     const differences = areDifferences(video, videos[0])
 274     if (differences !== undefined) {
 275       console.error('Integrity not ok with server %d!', i + 1)
 276
 277       if (displayDiffOnFail) {
 278         console.log(differences)
 279       }
 280
 281       process.exit(-1)
 282     }
 283
 284     i++
 285   }
 286
 287   console.log('Integrity ok.')
 288 }
 289
 290 function areDifferences (videos1: Video[], videos2: Video[]) {
 291   // Remove some keys we don't want to compare
 292   videos1.concat(videos2).forEach(video => {
 293     delete video.id
 294     delete video.isLocal
 295     delete video.thumbnailPath
 296     delete video.updatedAt
 297     delete video.views
 298   })
 299
 300   if (videos1.length !== videos2.length) {
 301     return `Videos length are different (${videos1.length}/${videos2.length}).`
 302   }
 303
 304   for (const video1 of videos1) {
 305     const video2 = videos2.find(video => video.uuid === video1.uuid)
 306
 307     if (!video2) return 'Video ' + video1.uuid + ' is missing.'
 308
 309     for (const videoKey of Object.keys(video1)) {
 310       const attribute1 = video1[videoKey]
 311       const attribute2 = video2[videoKey]
 312
 313       if (videoKey === 'tags') {
 314         if (attribute1.length !== attribute2.length) {
 315           return 'Tags are different.'
 316         }
 317
 318         attribute1.forEach(tag1 => {
 319           if (attribute2.indexOf(tag1) === -1) {
 320             return 'Tag ' + tag1 + ' is missing.'
 321           }
 322         })
 323       } else if (videoKey === 'files') {
 324         if (attribute1.length !== attribute2.length) {
 325           return 'Video files are different.'
 326         }
 327
 328         attribute1.forEach((videoFile1: VideoFile) => {
 329           const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri)
 330           if (!videoFile2) {
 331             return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.`
 332           }
 333
 334           if (videoFile1.size !== videoFile2.size || videoFile1.resolutionLabel !== videoFile2.resolutionLabel) {
 335             return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.`
 336           }
 337         })
 338       } else {
 339         if (attribute1 !== attribute2) {
 340           return `Video ${video1.uuid} has different value for attribute ${videoKey}.`
 341         }
 342       }
 343     }
 344   }
 345
 346   return undefined
 347 }
 348
 349 function goodbye () {
 350   return process.exit(-1)
 351 }
 352
 353 async function isTherePendingRequests (servers: ServerInfo[]) {
 354   const tasks: Promise<any>[] = []
 355   let pendingRequests = false
 356
 357   // Check if each server has pending request
 358   for (const server of servers) {
 359     const p = getRequestsStats(server).then(res => {
 360       const stats = res.body
 361
 362       if (
 363         stats.requestScheduler.totalRequests !== 0 ||
 364         stats.requestVideoEventScheduler.totalRequests !== 0 ||
 365         stats.requestVideoQaduScheduler.totalRequests !== 0
 366       ) {
 367         pendingRequests = true
 368       }
 369     })
 370
 371     tasks.push(p)
 372   }
 373
 374   await Promise.all(tasks)
 375
 376   return pendingRequests
 377 }