server/tests/real-world/real-world.ts

   1 import * as program from 'commander'
   2 import { isEqual, differenceWith } from 'lodash'
   3
   4 // /!\ Before imports /!\
   5 process.env.NODE_ENV = 'test'
   6
   7 import { REQUESTS_INTERVAL } from '../../initializers/constants'
   8 import { Video, VideoRateType } from '../../../shared'
   9 import {
  10   ServerInfo as DefaultServerInfo,
  11   flushAndRunMultipleServers,
  12   setAccessTokensToServers,
  13   makeFriends,
  14   wait,
  15   killallServers,
  16   flushTests,
  17   uploadVideo,
  18   getVideosList,
  19   updateVideo,
  20   removeVideo,
  21   getVideo,
  22   getAllVideosListBy,
  23   getRequestsStats
  24 } from '../utils'
  25
  26 interface ServerInfo extends DefaultServerInfo {
  27   requestsNumber: number
  28 }
  29
  30 program
  31   .option('-c, --create [weight]', 'Weight for creating videos')
  32   .option('-r, --remove [weight]', 'Weight for removing videos')
  33   .option('-u, --update [weight]', 'Weight for updating videos')
  34   .option('-v, --view [weight]', 'Weight for viewing videos')
  35   .option('-l, --like [weight]', 'Weight for liking videos')
  36   .option('-s, --dislike [weight]', 'Weight for disliking videos')
  37   .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3)
  38   .option('-i, --interval-action [interval]', 'Interval in ms for an action')
  39   .option('-I, --interval-integrity [interval]', 'Interval in ms for an integrity check')
  40   .option('-f, --flush', 'Flush datas on exit')
  41   .option('-d, --difference', 'Display difference if integrity is not okay')
  42   .parse(process.argv)
  43
  44 const createWeight = program['create'] !== undefined ? parseInt(program['create'], 10) : 5
  45 const removeWeight = program['remove'] !== undefined ? parseInt(program['remove'], 10) : 4
  46 const updateWeight = program['update'] !== undefined ? parseInt(program['update'], 10) : 4
  47 const viewWeight = program['view'] !== undefined ? parseInt(program['view'], 10) : 4
  48 const likeWeight = program['like'] !== undefined ? parseInt(program['like'], 10) : 4
  49 const dislikeWeight = program['dislike'] !== undefined ? parseInt(program['dislike'], 10) : 4
  50 const flushAtExit = program['flush'] || false
  51 const actionInterval = program['intervalAction'] !== undefined ? parseInt(program['intervalAction'], 10) : 500
  52 const integrityInterval = program['intervalIntegrity'] !== undefined ? parseInt(program['intervalIntegrity'], 10) : 60000
  53 const displayDiffOnFail = program['difference'] || false
  54
  55 const numberOfPods = 6
  56
  57 console.log(
  58   'Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.',
  59   createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight
  60 )
  61
  62 if (flushAtExit) {
  63   console.log('Program will flush data on exit.')
  64 } else {
  65   console.log('Program will not flush data on exit.')
  66 }
  67 if (displayDiffOnFail) {
  68   console.log('Program will display diff on failure.')
  69 } else {
  70   console.log('Program will not display diff on failure')
  71 }
  72 console.log('Interval in ms for each action: %d.', actionInterval)
  73 console.log('Interval in ms for each integrity check: %d.', integrityInterval)
  74
  75 console.log('Run servers...')
  76
  77 start()
  78
  79 // ----------------------------------------------------------------------------
  80
  81 async function start () {
  82   const servers = await runServers(numberOfPods)
  83
  84   process.on('exit', async () => await exitServers(servers, flushAtExit))
  85   process.on('SIGINT', goodbye)
  86   process.on('SIGTERM', goodbye)
  87
  88   console.log('Servers runned')
  89   initializeRequestsPerServer(servers)
  90
  91   let checking = false
  92
  93   setInterval(async () => {
  94     if (checking === true) return
  95
  96     const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight)
  97
  98     const numServer = getRandomNumServer(servers)
  99     servers[numServer].requestsNumber++
 100
 101     if (rand < createWeight) {
 102       await upload(servers, numServer)
 103     } else if (rand < createWeight + updateWeight) {
 104       await update(servers, numServer)
 105     } else if (rand < createWeight + updateWeight + removeWeight) {
 106       await remove(servers, numServer)
 107     } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) {
 108       await view(servers, numServer)
 109     } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) {
 110       await like(servers, numServer)
 111     } else {
 112       await dislike(servers, numServer)
 113     }
 114   }, actionInterval)
 115
 116   // The function will check the consistency between servers (should have the same videos with same attributes...)
 117   setInterval(function () {
 118     if (checking === true) return
 119
 120     console.log('Checking integrity...')
 121     checking = true
 122
 123     const waitingInterval = setInterval(async () => {
 124       const awaitingRequests = await isThereAwaitingRequests(servers)
 125       if (awaitingRequests === true) {
 126         console.log('A server has awaiting requests, waiting...')
 127         return
 128       }
 129
 130       await checkIntegrity(servers)
 131
 132       initializeRequestsPerServer(servers)
 133       checking = false
 134       clearInterval(waitingInterval)
 135     }, REQUESTS_INTERVAL)
 136   }, integrityInterval)
 137 }
 138
 139 function initializeRequestsPerServer (servers: ServerInfo[]) {
 140   servers.forEach(server => server.requestsNumber = 0)
 141 }
 142
 143 function getRandomInt (min, max) {
 144   return Math.floor(Math.random() * (max - min)) + min
 145 }
 146
 147 function getRandomNumServer (servers) {
 148   return getRandomInt(0, servers.length)
 149 }
 150
 151 async function runServers (numberOfPods: number) {
 152   let servers = null
 153
 154   // Run servers
 155   servers = await flushAndRunMultipleServers(numberOfPods)
 156
 157   // Get the access tokens
 158   await setAccessTokensToServers(servers)
 159
 160   await makeFriends(servers[1].url, servers[1].accessToken)
 161   await makeFriends(servers[0].url, servers[0].accessToken)
 162   await wait(1000)
 163
 164   await makeFriends(servers[3].url, servers[3].accessToken)
 165   await makeFriends(servers[5].url, servers[5].accessToken)
 166   await makeFriends(servers[4].url, servers[4].accessToken)
 167
 168   await wait(1000)
 169
 170   return servers
 171 }
 172
 173 async function exitServers (servers: ServerInfo[], flushAtExit: boolean) {
 174   killallServers(servers)
 175
 176   if (flushAtExit) await flushTests()
 177 }
 178
 179 function upload (servers: ServerInfo[], numServer: number) {
 180   console.log('Uploading video to server ' + numServer)
 181
 182   const videoAttributes = {
 183     name: Date.now() + ' name',
 184     category: 4,
 185     nsfw: false,
 186     licence: 2,
 187     language: 1,
 188     description: Date.now() + ' description',
 189     tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ],
 190     fixture: 'video_short1.webm'
 191   }
 192   return uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes)
 193 }
 194
 195 async function update (servers: ServerInfo[], numServer: number) {
 196   const res = await getVideosList(servers[numServer].url)
 197
 198   const videos = res.body.data.filter(video => video.isLocal === true)
 199   if (videos.length === 0) return undefined
 200
 201   const toUpdate = videos[getRandomInt(0, videos.length)].id
 202   const attributes = {
 203     name: Date.now() + ' name',
 204     description: Date.now() + ' description',
 205     tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ]
 206   }
 207
 208   console.log('Updating video of server ' + numServer)
 209
 210   return updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes)
 211 }
 212
 213 async function remove (servers: ServerInfo[], numServer: number) {
 214   const res = await getVideosList(servers[numServer].url)
 215   const videos = res.body.data
 216   if (videos.length === 0) return undefined
 217
 218   const toRemove = videos[getRandomInt(0, videos.length)].id
 219
 220   console.log('Removing video from server ' + numServer)
 221   return removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove)
 222 }
 223
 224 async function view (servers: ServerInfo[], numServer: number) {
 225   const res = await getVideosList(servers[numServer].url)
 226
 227   const videos = res.body.data
 228   if (videos.length === 0) return undefined
 229
 230   const toView = videos[getRandomInt(0, videos.length)].id
 231
 232   console.log('Viewing video from server ' + numServer)
 233   return getVideo(servers[numServer].url, toView)
 234 }
 235
 236 function like (servers: ServerInfo[], numServer: number) {
 237   return rate(servers, numServer, 'like')
 238 }
 239
 240 function dislike (servers: ServerInfo[], numServer: number) {
 241   return rate(servers, numServer, 'dislike')
 242 }
 243
 244 async function rate (servers: ServerInfo[], numServer: number, rating: VideoRateType) {
 245   const res = await getVideosList(servers[numServer].url)
 246
 247   const videos = res.body.data
 248   if (videos.length === 0) return undefined
 249
 250   const toRate = videos[getRandomInt(0, videos.length)].id
 251
 252   console.log('Rating (%s) video from server %d', rating, numServer)
 253   return getVideo(servers[numServer].url, toRate)
 254 }
 255
 256 async function checkIntegrity (servers: ServerInfo[]) {
 257   const videos: Video[][] = []
 258   const tasks: Promise<any>[] = []
 259
 260   // Fetch all videos and remove some fields that can differ between pods
 261   for (const server of servers) {
 262     const p = getAllVideosListBy(server.url).then(res => {
 263       const serverVideos = res.body.data
 264       for (const serverVideo of serverVideos) {
 265         delete serverVideo.id
 266         delete serverVideo.isLocal
 267         delete serverVideo.thumbnailPath
 268         delete serverVideo.updatedAt
 269         delete serverVideo.views
 270       }
 271
 272       videos.push(serverVideos)
 273     })
 274
 275     tasks.push(p)
 276   }
 277
 278   await Promise.all(tasks)
 279
 280   let i = 0
 281   for (const video of videos) {
 282     if (!isEqual(video, videos[0])) {
 283       console.error('Integrity not ok with server %d!', i + 1)
 284
 285       if (displayDiffOnFail) {
 286         console.log(differenceWith(videos[0], video, isEqual))
 287         console.log(differenceWith(video, videos[0], isEqual))
 288       }
 289
 290       process.exit(-1)
 291     }
 292
 293     i++
 294   }
 295
 296   console.log('Integrity ok.')
 297 }
 298
 299 function goodbye () {
 300   return process.exit(-1)
 301 }
 302
 303 async function isThereAwaitingRequests (servers: ServerInfo[]) {
 304   const tasks: Promise<any>[] = []
 305   let awaitingRequests = false
 306
 307   // Check if each server has awaiting request
 308   for (const server of servers) {
 309     const p = getRequestsStats(server).then(res => {
 310       const stats = res.body
 311
 312       if (
 313         stats.requestScheduler.totalRequests !== 0 ||
 314         stats.requestVideoEventScheduler.totalRequests !== 0 ||
 315         stats.requestVideoQaduScheduler.totalRequests !== 0
 316       ) {
 317         awaitingRequests = true
 318       }
 319     })
 320
 321     tasks.push(p)
 322   }
 323
 324   await Promise.all(tasks)
 325
 326   return awaitingRequests
 327 }