server/tests/real-world/real-world.js

   1 'use strict'
   2
   3 const each = require('async/each')
   4 const isEqual = require('lodash/isEqual')
   5 const differenceWith = require('lodash/differenceWith')
   6 const program = require('commander')
   7 const series = require('async/series')
   8
   9 process.env.NODE_ENV = 'test'
  10 const constants = require('../../initializers/constants')
  11
  12 const loginUtils = require('../utils/login')
  13 const podsUtils = require('../utils/pods')
  14 const serversUtils = require('../utils/servers')
  15 const videosUtils = require('../utils/videos')
  16 const requestStatsUtils = require('../utils/requests-stats')
  17
  18 program
  19   .option('-c, --create [weight]', 'Weight for creating videos')
  20   .option('-r, --remove [weight]', 'Weight for removing videos')
  21   .option('-u, --update [weight]', 'Weight for updating videos')
  22   .option('-v, --view [weight]', 'Weight for viewing videos')
  23   .option('-l, --like [weight]', 'Weight for liking videos')
  24   .option('-s --dislike [weight]', 'Weight for disliking videos')
  25   .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3)
  26   .option('-a, --action [interval]', 'Interval in ms for an action')
  27   .option('-i, --integrity [interval]', 'Interval in ms for an integrity check')
  28   .option('-f, --flush', 'Flush datas on exit')
  29   .option('-d, --difference', 'Display difference if integrity is not okay')
  30   .parse(process.argv)
  31
  32 const createWeight = program.create !== undefined ? parseInt(program.create) : 5
  33 const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4
  34 const updateWeight = program.update !== undefined ? parseInt(program.update) : 4
  35 const viewWeight = program.view !== undefined ? parseInt(program.view) : 4
  36 const likeWeight = program.like !== undefined ? parseInt(program.like) : 4
  37 const dislikeWeight = program.dislike !== undefined ? parseInt(program.dislike) : 4
  38 const flushAtExit = program.flush || false
  39 const actionInterval = program.action !== undefined ? parseInt(program.action) : 500
  40 const integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000
  41 const displayDiffOnFail = program.integrity || false
  42
  43 const numberOfPods = 6
  44
  45 console.log('Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.', createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight)
  46 if (flushAtExit) {
  47   console.log('Program will flush data on exit.')
  48 } else {
  49   console.log('Program will not flush data on exit.')
  50 }
  51 if (displayDiffOnFail) {
  52   console.log('Program will display diff on failure.')
  53 } else {
  54   console.log('Program will not display diff on failure')
  55 }
  56 console.log('Interval in ms for each action: %d.', actionInterval)
  57 console.log('Interval in ms for each integrity check: %d.', integrityInterval)
  58
  59 console.log('Run servers...')
  60 runServers(numberOfPods, function (err, servers) {
  61   if (err) throw err
  62
  63   process.on('exit', function () {
  64     exitServers(servers, flushAtExit)
  65   })
  66   process.on('SIGINT', goodbye)
  67   process.on('SIGTERM', goodbye)
  68
  69   console.log('Servers runned')
  70   initializeRequestsPerServer(servers)
  71
  72   let checking = false
  73
  74   setInterval(function () {
  75     if (checking === true) return
  76
  77     const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight)
  78
  79     const numServer = getRandomNumServer(servers)
  80     servers[numServer].requestsNumber++
  81
  82     if (rand < createWeight) {
  83       upload(servers, numServer)
  84     } else if (rand < createWeight + updateWeight) {
  85       update(servers, numServer)
  86     } else if (rand < createWeight + updateWeight + removeWeight) {
  87       remove(servers, numServer)
  88     } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) {
  89       view(servers, numServer)
  90     } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) {
  91       like(servers, numServer)
  92     } else {
  93       dislike(servers, numServer)
  94     }
  95   }, actionInterval)
  96
  97   // The function will check the consistency between servers (should have the same videos with same attributes...)
  98   setInterval(function () {
  99     if (checking === true) return
 100
 101     console.log('Checking integrity...')
 102     checking = true
 103
 104     const waitingInterval = setInterval(function () {
 105       isThereAwaitingRequests(servers, function (res) {
 106         if (res === true) {
 107           console.log('A server has awaiting requests, waiting...')
 108           return
 109         }
 110
 111         checkIntegrity(servers, function () {
 112           initializeRequestsPerServer(servers)
 113           checking = false
 114           clearInterval(waitingInterval)
 115         })
 116       })
 117     }, constants.REQUESTS_INTERVAL)
 118   }, integrityInterval)
 119 })
 120
 121 // ----------------------------------------------------------------------------
 122
 123 function initializeRequestsPerServer (servers) {
 124   servers.forEach(function (server) {
 125     server.requestsNumber = 0
 126   })
 127 }
 128
 129 function getRandomInt (min, max) {
 130   return Math.floor(Math.random() * (max - min)) + min
 131 }
 132
 133 function getRandomNumServer (servers) {
 134   return getRandomInt(0, servers.length)
 135 }
 136
 137 function runServers (numberOfPods, callback) {
 138   let servers = null
 139
 140   series([
 141     // Run servers
 142     function (next) {
 143       serversUtils.flushAndRunMultipleServers(numberOfPods, function (serversRun) {
 144         servers = serversRun
 145         next()
 146       })
 147     },
 148     // Get the access tokens
 149     function (next) {
 150       each(servers, function (server, callbackEach) {
 151         loginUtils.loginAndGetAccessToken(server, function (err, accessToken) {
 152           if (err) return callbackEach(err)
 153
 154           server.accessToken = accessToken
 155           callbackEach()
 156         })
 157       }, next)
 158     },
 159     function (next) {
 160       const server = servers[1]
 161       podsUtils.makeFriends(server.url, server.accessToken, next)
 162     },
 163     function (next) {
 164       const server = servers[0]
 165       podsUtils.makeFriends(server.url, server.accessToken, next)
 166     },
 167     function (next) {
 168       setTimeout(next, 1000)
 169     },
 170     function (next) {
 171       const server = servers[3]
 172       podsUtils.makeFriends(server.url, server.accessToken, next)
 173     },
 174     function (next) {
 175       const server = servers[5]
 176       podsUtils.makeFriends(server.url, server.accessToken, next)
 177     },
 178     function (next) {
 179       const server = servers[4]
 180       podsUtils.makeFriends(server.url, server.accessToken, next)
 181     },
 182     function (next) {
 183       setTimeout(next, 1000)
 184     }
 185   ], function (err) {
 186     return callback(err, servers)
 187   })
 188 }
 189
 190 function exitServers (servers, callback) {
 191   if (!callback) callback = function () {}
 192
 193   servers.forEach(function (server) {
 194     if (server.app) process.kill(-server.app.pid)
 195   })
 196
 197   if (flushAtExit) serversUtils.flushTests(callback)
 198 }
 199
 200 function upload (servers, numServer, callback) {
 201   if (!callback) callback = function () {}
 202
 203   console.log('Uploading video to server ' + numServer)
 204
 205   const videoAttributes = {
 206     name: Date.now() + ' name',
 207     category: 4,
 208     nsfw: false,
 209     licence: 2,
 210     description: Date.now() + ' description',
 211     tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ],
 212     fixture: 'video_short1.webm'
 213   }
 214   videosUtils.uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes, callback)
 215 }
 216
 217 function update (servers, numServer, callback) {
 218   if (!callback) callback = function () {}
 219
 220   videosUtils.getVideosList(servers[numServer].url, function (err, res) {
 221     if (err) throw err
 222
 223     const videos = res.body.data.filter(function (video) { return video.isLocal })
 224     if (videos.length === 0) return callback()
 225
 226     const toUpdate = videos[getRandomInt(0, videos.length)].id
 227     const attributes = {
 228       name: Date.now() + ' name',
 229       description: Date.now() + ' description',
 230       tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ]
 231     }
 232
 233     console.log('Updating video of server ' + numServer)
 234
 235     videosUtils.updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes, callback)
 236   })
 237 }
 238
 239 function remove (servers, numServer, callback) {
 240   if (!callback) callback = function () {}
 241
 242   videosUtils.getVideosList(servers[numServer].url, function (err, res) {
 243     if (err) throw err
 244
 245     const videos = res.body.data
 246     if (videos.length === 0) return callback()
 247
 248     const toRemove = videos[getRandomInt(0, videos.length)].id
 249
 250     console.log('Removing video from server ' + numServer)
 251     videosUtils.removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove, callback)
 252   })
 253 }
 254
 255 function view (servers, numServer, callback) {
 256   if (!callback) callback = function () {}
 257
 258   videosUtils.getVideosList(servers[numServer].url, function (err, res) {
 259     if (err) throw err
 260
 261     const videos = res.body.data
 262     if (videos.length === 0) return callback()
 263
 264     const toView = videos[getRandomInt(0, videos.length)].id
 265
 266     console.log('Viewing video from server ' + numServer)
 267     videosUtils.getVideo(servers[numServer].url, toView, callback)
 268   })
 269 }
 270
 271 function like (servers, numServer, callback) {
 272   rate(servers, numServer, 'like', callback)
 273 }
 274
 275 function dislike (servers, numServer, callback) {
 276   rate(servers, numServer, 'dislike', callback)
 277 }
 278
 279 function rate (servers, numServer, rating, callback) {
 280   if (!callback) callback = function () {}
 281
 282   videosUtils.getVideosList(servers[numServer].url, function (err, res) {
 283     if (err) throw err
 284
 285     const videos = res.body.data
 286     if (videos.length === 0) return callback()
 287
 288     const toRate = videos[getRandomInt(0, videos.length)].id
 289
 290     console.log('Rating (%s) video from server %d', rating, numServer)
 291     videosUtils.getVideo(servers[numServer].url, toRate, callback)
 292   })
 293 }
 294
 295 function checkIntegrity (servers, callback) {
 296   const videos = []
 297   each(servers, function (server, callback) {
 298     videosUtils.getAllVideosListBy(server.url, function (err, res) {
 299       if (err) throw err
 300       const serverVideos = res.body.data
 301       for (const serverVideo of serverVideos) {
 302         delete serverVideo.id
 303         delete serverVideo.isLocal
 304         delete serverVideo.thumbnailPath
 305         delete serverVideo.updatedAt
 306         delete serverVideo.views
 307       }
 308
 309       videos.push(serverVideos)
 310       callback()
 311     })
 312   }, function () {
 313     let i = 0
 314
 315     for (const video of videos) {
 316       if (!isEqual(video, videos[0])) {
 317         console.error('Integrity not ok with server %d!', i + 1)
 318
 319         if (displayDiffOnFail) {
 320           console.log(differenceWith(videos[0], video, isEqual))
 321           console.log(differenceWith(video, videos[0], isEqual))
 322         }
 323
 324         process.exit(-1)
 325       }
 326
 327       i++
 328     }
 329
 330     console.log('Integrity ok.')
 331     return callback()
 332   })
 333 }
 334
 335 function goodbye () {
 336   return process.exit(-1)
 337 }
 338
 339 function isThereAwaitingRequests (servers, callback) {
 340   let noRequests = true
 341
 342   // Check is each server has awaiting requestq
 343   each(servers, function (server, callbackEach) {
 344     requestStatsUtils.getRequestsStats(server, server.accessToken, function (err, res) {
 345       if (err) throw err
 346
 347       const stats = res.body
 348
 349       if (
 350         stats.requestScheduler.totalRequests !== 0 ||
 351         stats.requestVideoEventScheduler.totalRequests !== 0 ||
 352         stats.requestVideoQaduScheduler.totalRequests !== 0
 353       ) {
 354         noRequests = false
 355       }
 356
 357       callbackEach()
 358     })
 359   }, function (err) {
 360     if (err) throw err
 361
 362     return callback(noRequests === false)
 363   })
 364 }