X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Ftests%2Freal-world%2Freal-world.ts;h=a96469b11181be6782ebdf872e370401b06a9f69;hb=92b9d60c00432c58d6184f3683bdb14a0300a3c6;hp=a200bd02dbaae6881b202962df1f77018abb6e6a;hpb=096641566f2663076c3b0d5a1947ecb7efb29fc0;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts index a200bd02d..a96469b11 100644 --- a/server/tests/real-world/real-world.ts +++ b/server/tests/real-world/real-world.ts @@ -1,27 +1,23 @@ -import * as program from 'commander' -import { isEqual, differenceWith } from 'lodash' - // /!\ Before imports /!\ process.env.NODE_ENV = 'test' -import { REQUESTS_INTERVAL } from '../../initializers/constants' -import { Video, VideoRateType } from '../../../shared' +import * as program from 'commander' +import { Video, VideoFile, VideoRateType } from '../../../shared' +import { JobState } from '../../../shared/models' import { - ServerInfo as DefaultServerInfo, flushAndRunMultipleServers, - setAccessTokensToServers, - makeFriends, - wait, + flushTests, follow, + getVideo, + getVideosList, getVideosListPagination, killallServers, - flushTests, - uploadVideo, - getVideosList, - updateVideo, removeVideo, - getVideo, - getAllVideosListBy, - getRequestsStats + ServerInfo as DefaultServerInfo, + setAccessTokensToServers, + updateVideo, + uploadVideo, viewVideo, + wait } from '../utils' +import { getJobsListPaginationAndSort } from '../utils/server/jobs' interface ServerInfo extends DefaultServerInfo { requestsNumber: number @@ -34,10 +30,10 @@ program .option('-v, --view [weight]', 'Weight for viewing videos') .option('-l, --like [weight]', 'Weight for liking videos') .option('-s, --dislike [weight]', 'Weight for disliking videos') - .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3) + .option('-p, --servers [n]', 'Number of servers to run (3 or 6)', /^3|6$/, 3) .option('-i, --interval-action [interval]', 'Interval in ms for an action') .option('-I, --interval-integrity [interval]', 'Interval in ms for an integrity check') - .option('-f, --flush', 'Flush datas on exit') + .option('-f, --flush', 'Flush data on exit') .option('-d, --difference', 'Display difference if integrity is not okay') .parse(process.argv) @@ -52,7 +48,7 @@ const actionInterval = program['intervalAction'] !== undefined ? parseInt(progra const integrityInterval = program['intervalIntegrity'] !== undefined ? parseInt(program['intervalIntegrity'], 10) : 60000 const displayDiffOnFail = program['difference'] || false -const numberOfPods = 6 +const numberOfServers = 6 console.log( 'Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.', @@ -79,13 +75,17 @@ start() // ---------------------------------------------------------------------------- async function start () { - const servers = await runServers(numberOfPods) + const servers = await runServers(numberOfServers) - process.on('exit', async () => await exitServers(servers, flushAtExit)) + process.on('exit', async () => { + await exitServers(servers, flushAtExit) + + return + }) process.on('SIGINT', goodbye) process.on('SIGTERM', goodbye) - console.log('Servers runned') + console.log('Servers ran') initializeRequestsPerServer(servers) let checking = false @@ -121,18 +121,20 @@ async function start () { checking = true const waitingInterval = setInterval(async () => { - const awaitingRequests = await isThereAwaitingRequests(servers) - if (awaitingRequests === true) { - console.log('A server has awaiting requests, waiting...') + const pendingRequests = await isTherePendingRequests(servers) + if (pendingRequests === true) { + console.log('A server has pending requests, waiting...') return } + // Even if there are no pending request, wait some potential processes + await wait(2000) await checkIntegrity(servers) initializeRequestsPerServer(servers) checking = false clearInterval(waitingInterval) - }, REQUESTS_INTERVAL) + }, 10000) }, integrityInterval) } @@ -148,24 +150,20 @@ function getRandomNumServer (servers) { return getRandomInt(0, servers.length) } -async function runServers (numberOfPods: number) { - let servers = null - - // Run servers - servers = await flushAndRunMultipleServers(numberOfPods) +async function runServers (numberOfServers: number) { + const servers: ServerInfo[] = (await flushAndRunMultipleServers(numberOfServers)) + .map(s => Object.assign({ requestsNumber: 0 }, s)) // Get the access tokens await setAccessTokensToServers(servers) - await makeFriends(servers[1].url, servers[1].accessToken) - await makeFriends(servers[0].url, servers[0].accessToken) - await wait(1000) + for (let i = 0; i < numberOfServers; i++) { + for (let j = 0; j < numberOfServers; j++) { + if (i === j) continue - await makeFriends(servers[3].url, servers[3].accessToken) - await makeFriends(servers[5].url, servers[5].accessToken) - await makeFriends(servers[4].url, servers[4].accessToken) - - await wait(1000) + await follow(servers[i].url, [ servers[j].url ], servers[i].accessToken) + } + } return servers } @@ -184,7 +182,7 @@ function upload (servers: ServerInfo[], numServer: number) { category: 4, nsfw: false, licence: 2, - language: 1, + language: 'en', description: Date.now() + ' description', tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ], fixture: 'video_short1.webm' @@ -212,7 +210,7 @@ async function update (servers: ServerInfo[], numServer: number) { async function remove (servers: ServerInfo[], numServer: number) { const res = await getVideosList(servers[numServer].url) - const videos = res.body.data + const videos = res.body.data.filter(video => video.isLocal === true) if (videos.length === 0) return undefined const toRemove = videos[getRandomInt(0, videos.length)].id @@ -230,7 +228,7 @@ async function view (servers: ServerInfo[], numServer: number) { const toView = videos[getRandomInt(0, videos.length)].id console.log('Viewing video from server ' + numServer) - return getVideo(servers[numServer].url, toView) + return viewVideo(servers[numServer].url, toView) } function like (servers: ServerInfo[], numServer: number) { @@ -257,21 +255,10 @@ async function checkIntegrity (servers: ServerInfo[]) { const videos: Video[][] = [] const tasks: Promise[] = [] - // Fetch all videos and remove some fields that can differ between pods + // Fetch all videos and remove some fields that can differ between servers for (const server of servers) { - const p = getAllVideosListBy(server.url).then(res => { - const serverVideos = res.body.data - for (const serverVideo of serverVideos) { - delete serverVideo.id - delete serverVideo.isLocal - delete serverVideo.thumbnailPath - delete serverVideo.updatedAt - delete serverVideo.views - } - - videos.push(serverVideos) - }) - + const p = getVideosListPagination(server.url, 0, 1000000, '-createdAt') + .then(res => videos.push(res.body.data)) tasks.push(p) } @@ -279,12 +266,12 @@ async function checkIntegrity (servers: ServerInfo[]) { let i = 0 for (const video of videos) { - if (!isEqual(video, videos[0])) { + const differences = areDifferences(video, videos[0]) + if (differences !== undefined) { console.error('Integrity not ok with server %d!', i + 1) if (displayDiffOnFail) { - console.log(differenceWith(videos[0], video, isEqual)) - console.log(differenceWith(video, videos[0], isEqual)) + console.log(differences) } process.exit(-1) @@ -296,32 +283,86 @@ async function checkIntegrity (servers: ServerInfo[]) { console.log('Integrity ok.') } +function areDifferences (videos1: Video[], videos2: Video[]) { + // Remove some keys we don't want to compare + videos1.concat(videos2).forEach(video => { + delete video.id + delete video.isLocal + delete video.thumbnailPath + delete video.updatedAt + delete video.views + }) + + if (videos1.length !== videos2.length) { + return `Videos length are different (${videos1.length}/${videos2.length}).` + } + + for (const video1 of videos1) { + const video2 = videos2.find(video => video.uuid === video1.uuid) + + if (!video2) return 'Video ' + video1.uuid + ' is missing.' + + for (const videoKey of Object.keys(video1)) { + const attribute1 = video1[videoKey] + const attribute2 = video2[videoKey] + + if (videoKey === 'tags') { + if (attribute1.length !== attribute2.length) { + return 'Tags are different.' + } + + attribute1.forEach(tag1 => { + if (attribute2.indexOf(tag1) === -1) { + return 'Tag ' + tag1 + ' is missing.' + } + }) + } else if (videoKey === 'files') { + if (attribute1.length !== attribute2.length) { + return 'Video files are different.' + } + + attribute1.forEach((videoFile1: VideoFile) => { + const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri) + if (!videoFile2) { + return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.` + } + + if (videoFile1.size !== videoFile2.size || videoFile1.resolution.label !== videoFile2.resolution.label) { + return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.` + } + }) + } else { + if (attribute1 !== attribute2) { + return `Video ${video1.uuid} has different value for attribute ${videoKey}.` + } + } + } + } + + return undefined +} + function goodbye () { return process.exit(-1) } -async function isThereAwaitingRequests (servers: ServerInfo[]) { +async function isTherePendingRequests (servers: ServerInfo[]) { + const states: JobState[] = [ 'waiting', 'active', 'delayed' ] const tasks: Promise[] = [] - let awaitingRequests = false + let pendingRequests = false - // Check if each server has awaiting request + // Check if each server has pending request for (const server of servers) { - const p = getRequestsStats(server).then(res => { - const stats = res.body - - if ( - stats.requestScheduler.totalRequests !== 0 || - stats.requestVideoEventScheduler.totalRequests !== 0 || - stats.requestVideoQaduScheduler.totalRequests !== 0 - ) { - awaitingRequests = true - } - }) - - tasks.push(p) + for (const state of states) { + const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') + .then(res => { + if (res.body.total > 0) pendingRequests = true + }) + tasks.push(p) + } } await Promise.all(tasks) - return awaitingRequests + return pendingRequests }