diff options
Diffstat (limited to 'server/tests/real-world/real-world.ts')
-rw-r--r-- | server/tests/real-world/real-world.ts | 375 |
1 files changed, 0 insertions, 375 deletions
diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts deleted file mode 100644 index cba5ac311..000000000 --- a/server/tests/real-world/real-world.ts +++ /dev/null | |||
@@ -1,375 +0,0 @@ | |||
1 | // /!\ Before imports /!\ | ||
2 | process.env.NODE_ENV = 'test' | ||
3 | |||
4 | import * as program from 'commander' | ||
5 | import { Video, VideoFile, VideoRateType } from '../../../shared' | ||
6 | import { JobState } from '../../../shared/models' | ||
7 | import { | ||
8 | flushAndRunMultipleServers, | ||
9 | flushTests, follow, | ||
10 | getVideo, | ||
11 | getVideosList, getVideosListPagination, | ||
12 | killallServers, | ||
13 | removeVideo, | ||
14 | ServerInfo as DefaultServerInfo, | ||
15 | setAccessTokensToServers, | ||
16 | updateVideo, | ||
17 | uploadVideo, viewVideo, | ||
18 | wait | ||
19 | } from '../../../shared/extra-utils' | ||
20 | import { getJobsListPaginationAndSort } from '../../../shared/extra-utils/server/jobs' | ||
21 | |||
22 | interface ServerInfo extends DefaultServerInfo { | ||
23 | requestsNumber: number | ||
24 | } | ||
25 | |||
26 | program | ||
27 | .option('-c, --create [weight]', 'Weight for creating videos') | ||
28 | .option('-r, --remove [weight]', 'Weight for removing videos') | ||
29 | .option('-u, --update [weight]', 'Weight for updating videos') | ||
30 | .option('-v, --view [weight]', 'Weight for viewing videos') | ||
31 | .option('-l, --like [weight]', 'Weight for liking videos') | ||
32 | .option('-s, --dislike [weight]', 'Weight for disliking videos') | ||
33 | .option('-p, --servers [n]', 'Number of servers to run (3 or 6)', /^3|6$/, 3) | ||
34 | .option('-i, --interval-action [interval]', 'Interval in ms for an action') | ||
35 | .option('-I, --interval-integrity [interval]', 'Interval in ms for an integrity check') | ||
36 | .option('-f, --flush', 'Flush data on exit') | ||
37 | .option('-d, --difference', 'Display difference if integrity is not okay') | ||
38 | .parse(process.argv) | ||
39 | |||
40 | const createWeight = program['create'] !== undefined ? parseInt(program['create'], 10) : 5 | ||
41 | const removeWeight = program['remove'] !== undefined ? parseInt(program['remove'], 10) : 4 | ||
42 | const updateWeight = program['update'] !== undefined ? parseInt(program['update'], 10) : 4 | ||
43 | const viewWeight = program['view'] !== undefined ? parseInt(program['view'], 10) : 4 | ||
44 | const likeWeight = program['like'] !== undefined ? parseInt(program['like'], 10) : 4 | ||
45 | const dislikeWeight = program['dislike'] !== undefined ? parseInt(program['dislike'], 10) : 4 | ||
46 | const flushAtExit = program['flush'] || false | ||
47 | const actionInterval = program['intervalAction'] !== undefined ? parseInt(program['intervalAction'], 10) : 500 | ||
48 | const integrityInterval = program['intervalIntegrity'] !== undefined ? parseInt(program['intervalIntegrity'], 10) : 60000 | ||
49 | const displayDiffOnFail = program['difference'] || false | ||
50 | |||
51 | const numberOfServers = 6 | ||
52 | |||
53 | console.log( | ||
54 | 'Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.', | ||
55 | createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight | ||
56 | ) | ||
57 | |||
58 | if (flushAtExit) { | ||
59 | console.log('Program will flush data on exit.') | ||
60 | } else { | ||
61 | console.log('Program will not flush data on exit.') | ||
62 | } | ||
63 | if (displayDiffOnFail) { | ||
64 | console.log('Program will display diff on failure.') | ||
65 | } else { | ||
66 | console.log('Program will not display diff on failure') | ||
67 | } | ||
68 | console.log('Interval in ms for each action: %d.', actionInterval) | ||
69 | console.log('Interval in ms for each integrity check: %d.', integrityInterval) | ||
70 | |||
71 | console.log('Run servers...') | ||
72 | |||
73 | start() | ||
74 | |||
75 | // ---------------------------------------------------------------------------- | ||
76 | |||
77 | async function start () { | ||
78 | const servers = await runServers(numberOfServers) | ||
79 | |||
80 | process.on('exit', async () => { | ||
81 | await exitServers(servers, flushAtExit) | ||
82 | |||
83 | return | ||
84 | }) | ||
85 | process.on('SIGINT', goodbye) | ||
86 | process.on('SIGTERM', goodbye) | ||
87 | |||
88 | console.log('Servers ran') | ||
89 | initializeRequestsPerServer(servers) | ||
90 | |||
91 | let checking = false | ||
92 | |||
93 | setInterval(async () => { | ||
94 | if (checking === true) return | ||
95 | |||
96 | const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight) | ||
97 | |||
98 | const numServer = getRandomNumServer(servers) | ||
99 | servers[numServer].requestsNumber++ | ||
100 | |||
101 | if (rand < createWeight) { | ||
102 | await upload(servers, numServer) | ||
103 | } else if (rand < createWeight + updateWeight) { | ||
104 | await update(servers, numServer) | ||
105 | } else if (rand < createWeight + updateWeight + removeWeight) { | ||
106 | await remove(servers, numServer) | ||
107 | } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) { | ||
108 | await view(servers, numServer) | ||
109 | } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) { | ||
110 | await like(servers, numServer) | ||
111 | } else { | ||
112 | await dislike(servers, numServer) | ||
113 | } | ||
114 | }, actionInterval) | ||
115 | |||
116 | // The function will check the consistency between servers (should have the same videos with same attributes...) | ||
117 | setInterval(function () { | ||
118 | if (checking === true) return | ||
119 | |||
120 | console.log('Checking integrity...') | ||
121 | checking = true | ||
122 | |||
123 | const waitingInterval = setInterval(async () => { | ||
124 | const pendingRequests = await isTherePendingRequests(servers) | ||
125 | if (pendingRequests === true) { | ||
126 | console.log('A server has pending requests, waiting...') | ||
127 | return | ||
128 | } | ||
129 | |||
130 | // Even if there are no pending request, wait some potential processes | ||
131 | await wait(2000) | ||
132 | await checkIntegrity(servers) | ||
133 | |||
134 | initializeRequestsPerServer(servers) | ||
135 | checking = false | ||
136 | clearInterval(waitingInterval) | ||
137 | }, 10000) | ||
138 | }, integrityInterval) | ||
139 | } | ||
140 | |||
141 | function initializeRequestsPerServer (servers: ServerInfo[]) { | ||
142 | servers.forEach(server => server.requestsNumber = 0) | ||
143 | } | ||
144 | |||
145 | function getRandomInt (min, max) { | ||
146 | return Math.floor(Math.random() * (max - min)) + min | ||
147 | } | ||
148 | |||
149 | function getRandomNumServer (servers) { | ||
150 | return getRandomInt(0, servers.length) | ||
151 | } | ||
152 | |||
153 | async function runServers (numberOfServers: number) { | ||
154 | const servers: ServerInfo[] = (await flushAndRunMultipleServers(numberOfServers)) | ||
155 | .map(s => Object.assign({ requestsNumber: 0 }, s)) | ||
156 | |||
157 | // Get the access tokens | ||
158 | await setAccessTokensToServers(servers) | ||
159 | |||
160 | for (let i = 0; i < numberOfServers; i++) { | ||
161 | for (let j = 0; j < numberOfServers; j++) { | ||
162 | if (i === j) continue | ||
163 | |||
164 | await follow(servers[i].url, [ servers[j].url ], servers[i].accessToken) | ||
165 | } | ||
166 | } | ||
167 | |||
168 | return servers | ||
169 | } | ||
170 | |||
171 | async function exitServers (servers: ServerInfo[], flushAtExit: boolean) { | ||
172 | killallServers(servers) | ||
173 | |||
174 | if (flushAtExit) await flushTests() | ||
175 | } | ||
176 | |||
177 | function upload (servers: ServerInfo[], numServer: number) { | ||
178 | console.log('Uploading video to server ' + numServer) | ||
179 | |||
180 | const videoAttributes = { | ||
181 | name: Date.now() + ' name', | ||
182 | category: 4, | ||
183 | nsfw: false, | ||
184 | licence: 2, | ||
185 | language: 'en', | ||
186 | description: Date.now() + ' description', | ||
187 | tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ], | ||
188 | fixture: 'video_short1.webm' | ||
189 | } | ||
190 | return uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes) | ||
191 | } | ||
192 | |||
193 | async function update (servers: ServerInfo[], numServer: number) { | ||
194 | const res = await getVideosList(servers[numServer].url) | ||
195 | |||
196 | const videos = res.body.data.filter(video => video.isLocal === true) | ||
197 | if (videos.length === 0) return undefined | ||
198 | |||
199 | const toUpdate = videos[getRandomInt(0, videos.length)].id | ||
200 | const attributes = { | ||
201 | name: Date.now() + ' name', | ||
202 | description: Date.now() + ' description', | ||
203 | tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] | ||
204 | } | ||
205 | |||
206 | console.log('Updating video of server ' + numServer) | ||
207 | |||
208 | return updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes) | ||
209 | } | ||
210 | |||
211 | async function remove (servers: ServerInfo[], numServer: number) { | ||
212 | const res = await getVideosList(servers[numServer].url) | ||
213 | const videos = res.body.data.filter(video => video.isLocal === true) | ||
214 | if (videos.length === 0) return undefined | ||
215 | |||
216 | const toRemove = videos[getRandomInt(0, videos.length)].id | ||
217 | |||
218 | console.log('Removing video from server ' + numServer) | ||
219 | return removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove) | ||
220 | } | ||
221 | |||
222 | async function view (servers: ServerInfo[], numServer: number) { | ||
223 | const res = await getVideosList(servers[numServer].url) | ||
224 | |||
225 | const videos = res.body.data | ||
226 | if (videos.length === 0) return undefined | ||
227 | |||
228 | const toView = videos[getRandomInt(0, videos.length)].id | ||
229 | |||
230 | console.log('Viewing video from server ' + numServer) | ||
231 | return viewVideo(servers[numServer].url, toView) | ||
232 | } | ||
233 | |||
234 | function like (servers: ServerInfo[], numServer: number) { | ||
235 | return rate(servers, numServer, 'like') | ||
236 | } | ||
237 | |||
238 | function dislike (servers: ServerInfo[], numServer: number) { | ||
239 | return rate(servers, numServer, 'dislike') | ||
240 | } | ||
241 | |||
242 | async function rate (servers: ServerInfo[], numServer: number, rating: VideoRateType) { | ||
243 | const res = await getVideosList(servers[numServer].url) | ||
244 | |||
245 | const videos = res.body.data | ||
246 | if (videos.length === 0) return undefined | ||
247 | |||
248 | const toRate = videos[getRandomInt(0, videos.length)].id | ||
249 | |||
250 | console.log('Rating (%s) video from server %d', rating, numServer) | ||
251 | return getVideo(servers[numServer].url, toRate) | ||
252 | } | ||
253 | |||
254 | async function checkIntegrity (servers: ServerInfo[]) { | ||
255 | const videos: Video[][] = [] | ||
256 | const tasks: Promise<any>[] = [] | ||
257 | |||
258 | // Fetch all videos and remove some fields that can differ between servers | ||
259 | for (const server of servers) { | ||
260 | const p = getVideosListPagination(server.url, 0, 1000000, '-createdAt') | ||
261 | .then(res => videos.push(res.body.data)) | ||
262 | tasks.push(p) | ||
263 | } | ||
264 | |||
265 | await Promise.all(tasks) | ||
266 | |||
267 | let i = 0 | ||
268 | for (const video of videos) { | ||
269 | const differences = areDifferences(video, videos[0]) | ||
270 | if (differences !== undefined) { | ||
271 | console.error('Integrity not ok with server %d!', i + 1) | ||
272 | |||
273 | if (displayDiffOnFail) { | ||
274 | console.log(differences) | ||
275 | } | ||
276 | |||
277 | process.exit(-1) | ||
278 | } | ||
279 | |||
280 | i++ | ||
281 | } | ||
282 | |||
283 | console.log('Integrity ok.') | ||
284 | } | ||
285 | |||
286 | function areDifferences (videos1: Video[], videos2: Video[]) { | ||
287 | // Remove some keys we don't want to compare | ||
288 | videos1.concat(videos2).forEach(video => { | ||
289 | delete video.id | ||
290 | delete video.isLocal | ||
291 | delete video.thumbnailPath | ||
292 | delete video.updatedAt | ||
293 | delete video.views | ||
294 | }) | ||
295 | |||
296 | if (videos1.length !== videos2.length) { | ||
297 | return `Videos length are different (${videos1.length}/${videos2.length}).` | ||
298 | } | ||
299 | |||
300 | for (const video1 of videos1) { | ||
301 | const video2 = videos2.find(video => video.uuid === video1.uuid) | ||
302 | |||
303 | if (!video2) return 'Video ' + video1.uuid + ' is missing.' | ||
304 | |||
305 | for (const videoKey of Object.keys(video1)) { | ||
306 | const attribute1 = video1[videoKey] | ||
307 | const attribute2 = video2[videoKey] | ||
308 | |||
309 | if (videoKey === 'tags') { | ||
310 | if (attribute1.length !== attribute2.length) { | ||
311 | return 'Tags are different.' | ||
312 | } | ||
313 | |||
314 | attribute1.forEach(tag1 => { | ||
315 | if (attribute2.indexOf(tag1) === -1) { | ||
316 | return 'Tag ' + tag1 + ' is missing.' | ||
317 | } | ||
318 | }) | ||
319 | } else if (videoKey === 'files') { | ||
320 | if (attribute1.length !== attribute2.length) { | ||
321 | return 'Video files are different.' | ||
322 | } | ||
323 | |||
324 | attribute1.forEach((videoFile1: VideoFile) => { | ||
325 | const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri) | ||
326 | if (!videoFile2) { | ||
327 | return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.` | ||
328 | } | ||
329 | |||
330 | if (videoFile1.size !== videoFile2.size || videoFile1.resolution.label !== videoFile2.resolution.label) { | ||
331 | return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.` | ||
332 | } | ||
333 | }) | ||
334 | } else { | ||
335 | if (attribute1 !== attribute2) { | ||
336 | return `Video ${video1.uuid} has different value for attribute ${videoKey}.` | ||
337 | } | ||
338 | } | ||
339 | } | ||
340 | } | ||
341 | |||
342 | return undefined | ||
343 | } | ||
344 | |||
345 | function goodbye () { | ||
346 | return process.exit(-1) | ||
347 | } | ||
348 | |||
349 | async function isTherePendingRequests (servers: ServerInfo[]) { | ||
350 | const states: JobState[] = [ 'waiting', 'active', 'delayed' ] | ||
351 | const tasks: Promise<any>[] = [] | ||
352 | let pendingRequests = false | ||
353 | |||
354 | // Check if each server has pending request | ||
355 | for (const server of servers) { | ||
356 | for (const state of states) { | ||
357 | const p = getJobsListPaginationAndSort({ | ||
358 | url: server.url, | ||
359 | accessToken: server.accessToken, | ||
360 | state: state, | ||
361 | start: 0, | ||
362 | count: 10, | ||
363 | sort: '-createdAt' | ||
364 | }) | ||
365 | .then(res => { | ||
366 | if (res.body.total > 0) pendingRequests = true | ||
367 | }) | ||
368 | tasks.push(p) | ||
369 | } | ||
370 | } | ||
371 | |||
372 | await Promise.all(tasks) | ||
373 | |||
374 | return pendingRequests | ||
375 | } | ||