]>
Commit | Line | Data |
---|---|---|
1 | // /!\ Before imports /!\ | |
2 | process.env.NODE_ENV = 'test' | |
3 | ||
4 | import * as program from 'commander' | |
5 | import { Video, VideoFile, VideoRateType } from '../../../shared' | |
6 | import { | |
7 | flushAndRunMultipleServers, | |
8 | flushTests, follow, | |
9 | getVideo, | |
10 | getVideosList, getVideosListPagination, | |
11 | killallServers, | |
12 | removeVideo, | |
13 | ServerInfo as DefaultServerInfo, | |
14 | setAccessTokensToServers, | |
15 | updateVideo, | |
16 | uploadVideo, viewVideo, | |
17 | wait | |
18 | } from '../utils' | |
19 | import { getJobsListPaginationAndSort } from '../utils/server/jobs' | |
20 | ||
21 | interface ServerInfo extends DefaultServerInfo { | |
22 | requestsNumber: number | |
23 | } | |
24 | ||
25 | program | |
26 | .option('-c, --create [weight]', 'Weight for creating videos') | |
27 | .option('-r, --remove [weight]', 'Weight for removing videos') | |
28 | .option('-u, --update [weight]', 'Weight for updating videos') | |
29 | .option('-v, --view [weight]', 'Weight for viewing videos') | |
30 | .option('-l, --like [weight]', 'Weight for liking videos') | |
31 | .option('-s, --dislike [weight]', 'Weight for disliking videos') | |
32 | .option('-p, --servers [n]', 'Number of servers to run (3 or 6)', /^3|6$/, 3) | |
33 | .option('-i, --interval-action [interval]', 'Interval in ms for an action') | |
34 | .option('-I, --interval-integrity [interval]', 'Interval in ms for an integrity check') | |
35 | .option('-f, --flush', 'Flush data on exit') | |
36 | .option('-d, --difference', 'Display difference if integrity is not okay') | |
37 | .parse(process.argv) | |
38 | ||
39 | const createWeight = program['create'] !== undefined ? parseInt(program['create'], 10) : 5 | |
40 | const removeWeight = program['remove'] !== undefined ? parseInt(program['remove'], 10) : 4 | |
41 | const updateWeight = program['update'] !== undefined ? parseInt(program['update'], 10) : 4 | |
42 | const viewWeight = program['view'] !== undefined ? parseInt(program['view'], 10) : 4 | |
43 | const likeWeight = program['like'] !== undefined ? parseInt(program['like'], 10) : 4 | |
44 | const dislikeWeight = program['dislike'] !== undefined ? parseInt(program['dislike'], 10) : 4 | |
45 | const flushAtExit = program['flush'] || false | |
46 | const actionInterval = program['intervalAction'] !== undefined ? parseInt(program['intervalAction'], 10) : 500 | |
47 | const integrityInterval = program['intervalIntegrity'] !== undefined ? parseInt(program['intervalIntegrity'], 10) : 60000 | |
48 | const displayDiffOnFail = program['difference'] || false | |
49 | ||
50 | const numberOfServers = 6 | |
51 | ||
52 | console.log( | |
53 | 'Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.', | |
54 | createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight | |
55 | ) | |
56 | ||
57 | if (flushAtExit) { | |
58 | console.log('Program will flush data on exit.') | |
59 | } else { | |
60 | console.log('Program will not flush data on exit.') | |
61 | } | |
62 | if (displayDiffOnFail) { | |
63 | console.log('Program will display diff on failure.') | |
64 | } else { | |
65 | console.log('Program will not display diff on failure') | |
66 | } | |
67 | console.log('Interval in ms for each action: %d.', actionInterval) | |
68 | console.log('Interval in ms for each integrity check: %d.', integrityInterval) | |
69 | ||
70 | console.log('Run servers...') | |
71 | ||
72 | start() | |
73 | ||
74 | // ---------------------------------------------------------------------------- | |
75 | ||
76 | async function start () { | |
77 | const servers = await runServers(numberOfServers) | |
78 | ||
79 | process.on('exit', async () => { | |
80 | await exitServers(servers, flushAtExit) | |
81 | ||
82 | return | |
83 | }) | |
84 | process.on('SIGINT', goodbye) | |
85 | process.on('SIGTERM', goodbye) | |
86 | ||
87 | console.log('Servers ran') | |
88 | initializeRequestsPerServer(servers) | |
89 | ||
90 | let checking = false | |
91 | ||
92 | setInterval(async () => { | |
93 | if (checking === true) return | |
94 | ||
95 | const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight) | |
96 | ||
97 | const numServer = getRandomNumServer(servers) | |
98 | servers[numServer].requestsNumber++ | |
99 | ||
100 | if (rand < createWeight) { | |
101 | await upload(servers, numServer) | |
102 | } else if (rand < createWeight + updateWeight) { | |
103 | await update(servers, numServer) | |
104 | } else if (rand < createWeight + updateWeight + removeWeight) { | |
105 | await remove(servers, numServer) | |
106 | } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) { | |
107 | await view(servers, numServer) | |
108 | } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) { | |
109 | await like(servers, numServer) | |
110 | } else { | |
111 | await dislike(servers, numServer) | |
112 | } | |
113 | }, actionInterval) | |
114 | ||
115 | // The function will check the consistency between servers (should have the same videos with same attributes...) | |
116 | setInterval(function () { | |
117 | if (checking === true) return | |
118 | ||
119 | console.log('Checking integrity...') | |
120 | checking = true | |
121 | ||
122 | const waitingInterval = setInterval(async () => { | |
123 | const pendingRequests = await isTherePendingRequests(servers) | |
124 | if (pendingRequests === true) { | |
125 | console.log('A server has pending requests, waiting...') | |
126 | return | |
127 | } | |
128 | ||
129 | // Even if there are no pending request, wait some potential processes | |
130 | await wait(2000) | |
131 | await checkIntegrity(servers) | |
132 | ||
133 | initializeRequestsPerServer(servers) | |
134 | checking = false | |
135 | clearInterval(waitingInterval) | |
136 | }, 10000) | |
137 | }, integrityInterval) | |
138 | } | |
139 | ||
140 | function initializeRequestsPerServer (servers: ServerInfo[]) { | |
141 | servers.forEach(server => server.requestsNumber = 0) | |
142 | } | |
143 | ||
144 | function getRandomInt (min, max) { | |
145 | return Math.floor(Math.random() * (max - min)) + min | |
146 | } | |
147 | ||
148 | function getRandomNumServer (servers) { | |
149 | return getRandomInt(0, servers.length) | |
150 | } | |
151 | ||
152 | async function runServers (numberOfServers: number) { | |
153 | const servers: ServerInfo[] = (await flushAndRunMultipleServers(numberOfServers)) | |
154 | .map(s => Object.assign({ requestsNumber: 0 }, s)) | |
155 | ||
156 | // Get the access tokens | |
157 | await setAccessTokensToServers(servers) | |
158 | ||
159 | for (let i = 0; i < numberOfServers; i++) { | |
160 | for (let j = 0; j < numberOfServers; j++) { | |
161 | if (i === j) continue | |
162 | ||
163 | await follow(servers[i].url, [ servers[j].url ], servers[i].accessToken) | |
164 | } | |
165 | } | |
166 | ||
167 | return servers | |
168 | } | |
169 | ||
170 | async function exitServers (servers: ServerInfo[], flushAtExit: boolean) { | |
171 | killallServers(servers) | |
172 | ||
173 | if (flushAtExit) await flushTests() | |
174 | } | |
175 | ||
176 | function upload (servers: ServerInfo[], numServer: number) { | |
177 | console.log('Uploading video to server ' + numServer) | |
178 | ||
179 | const videoAttributes = { | |
180 | name: Date.now() + ' name', | |
181 | category: 4, | |
182 | nsfw: false, | |
183 | licence: 2, | |
184 | language: 1, | |
185 | description: Date.now() + ' description', | |
186 | tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ], | |
187 | fixture: 'video_short1.webm' | |
188 | } | |
189 | return uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes) | |
190 | } | |
191 | ||
192 | async function update (servers: ServerInfo[], numServer: number) { | |
193 | const res = await getVideosList(servers[numServer].url) | |
194 | ||
195 | const videos = res.body.data.filter(video => video.isLocal === true) | |
196 | if (videos.length === 0) return undefined | |
197 | ||
198 | const toUpdate = videos[getRandomInt(0, videos.length)].id | |
199 | const attributes = { | |
200 | name: Date.now() + ' name', | |
201 | description: Date.now() + ' description', | |
202 | tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] | |
203 | } | |
204 | ||
205 | console.log('Updating video of server ' + numServer) | |
206 | ||
207 | return updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes) | |
208 | } | |
209 | ||
210 | async function remove (servers: ServerInfo[], numServer: number) { | |
211 | const res = await getVideosList(servers[numServer].url) | |
212 | const videos = res.body.data.filter(video => video.isLocal === true) | |
213 | if (videos.length === 0) return undefined | |
214 | ||
215 | const toRemove = videos[getRandomInt(0, videos.length)].id | |
216 | ||
217 | console.log('Removing video from server ' + numServer) | |
218 | return removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove) | |
219 | } | |
220 | ||
221 | async function view (servers: ServerInfo[], numServer: number) { | |
222 | const res = await getVideosList(servers[numServer].url) | |
223 | ||
224 | const videos = res.body.data | |
225 | if (videos.length === 0) return undefined | |
226 | ||
227 | const toView = videos[getRandomInt(0, videos.length)].id | |
228 | ||
229 | console.log('Viewing video from server ' + numServer) | |
230 | return viewVideo(servers[numServer].url, toView) | |
231 | } | |
232 | ||
233 | function like (servers: ServerInfo[], numServer: number) { | |
234 | return rate(servers, numServer, 'like') | |
235 | } | |
236 | ||
237 | function dislike (servers: ServerInfo[], numServer: number) { | |
238 | return rate(servers, numServer, 'dislike') | |
239 | } | |
240 | ||
241 | async function rate (servers: ServerInfo[], numServer: number, rating: VideoRateType) { | |
242 | const res = await getVideosList(servers[numServer].url) | |
243 | ||
244 | const videos = res.body.data | |
245 | if (videos.length === 0) return undefined | |
246 | ||
247 | const toRate = videos[getRandomInt(0, videos.length)].id | |
248 | ||
249 | console.log('Rating (%s) video from server %d', rating, numServer) | |
250 | return getVideo(servers[numServer].url, toRate) | |
251 | } | |
252 | ||
253 | async function checkIntegrity (servers: ServerInfo[]) { | |
254 | const videos: Video[][] = [] | |
255 | const tasks: Promise<any>[] = [] | |
256 | ||
257 | // Fetch all videos and remove some fields that can differ between servers | |
258 | for (const server of servers) { | |
259 | const p = getVideosListPagination(server.url, 0, 1000000, '-createdAt') | |
260 | .then(res => videos.push(res.body.data)) | |
261 | tasks.push(p) | |
262 | } | |
263 | ||
264 | await Promise.all(tasks) | |
265 | ||
266 | let i = 0 | |
267 | for (const video of videos) { | |
268 | const differences = areDifferences(video, videos[0]) | |
269 | if (differences !== undefined) { | |
270 | console.error('Integrity not ok with server %d!', i + 1) | |
271 | ||
272 | if (displayDiffOnFail) { | |
273 | console.log(differences) | |
274 | } | |
275 | ||
276 | process.exit(-1) | |
277 | } | |
278 | ||
279 | i++ | |
280 | } | |
281 | ||
282 | console.log('Integrity ok.') | |
283 | } | |
284 | ||
285 | function areDifferences (videos1: Video[], videos2: Video[]) { | |
286 | // Remove some keys we don't want to compare | |
287 | videos1.concat(videos2).forEach(video => { | |
288 | delete video.id | |
289 | delete video.isLocal | |
290 | delete video.thumbnailPath | |
291 | delete video.updatedAt | |
292 | delete video.views | |
293 | }) | |
294 | ||
295 | if (videos1.length !== videos2.length) { | |
296 | return `Videos length are different (${videos1.length}/${videos2.length}).` | |
297 | } | |
298 | ||
299 | for (const video1 of videos1) { | |
300 | const video2 = videos2.find(video => video.uuid === video1.uuid) | |
301 | ||
302 | if (!video2) return 'Video ' + video1.uuid + ' is missing.' | |
303 | ||
304 | for (const videoKey of Object.keys(video1)) { | |
305 | const attribute1 = video1[videoKey] | |
306 | const attribute2 = video2[videoKey] | |
307 | ||
308 | if (videoKey === 'tags') { | |
309 | if (attribute1.length !== attribute2.length) { | |
310 | return 'Tags are different.' | |
311 | } | |
312 | ||
313 | attribute1.forEach(tag1 => { | |
314 | if (attribute2.indexOf(tag1) === -1) { | |
315 | return 'Tag ' + tag1 + ' is missing.' | |
316 | } | |
317 | }) | |
318 | } else if (videoKey === 'files') { | |
319 | if (attribute1.length !== attribute2.length) { | |
320 | return 'Video files are different.' | |
321 | } | |
322 | ||
323 | attribute1.forEach((videoFile1: VideoFile) => { | |
324 | const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri) | |
325 | if (!videoFile2) { | |
326 | return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.` | |
327 | } | |
328 | ||
329 | if (videoFile1.size !== videoFile2.size || videoFile1.resolutionLabel !== videoFile2.resolutionLabel) { | |
330 | return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.` | |
331 | } | |
332 | }) | |
333 | } else { | |
334 | if (attribute1 !== attribute2) { | |
335 | return `Video ${video1.uuid} has different value for attribute ${videoKey}.` | |
336 | } | |
337 | } | |
338 | } | |
339 | } | |
340 | ||
341 | return undefined | |
342 | } | |
343 | ||
344 | function goodbye () { | |
345 | return process.exit(-1) | |
346 | } | |
347 | ||
348 | async function isTherePendingRequests (servers: ServerInfo[]) { | |
349 | const tasks: Promise<any>[] = [] | |
350 | let pendingRequests = false | |
351 | ||
352 | // Check if each server has pending request | |
353 | for (const server of servers) { | |
354 | const p = getJobsListPaginationAndSort(server.url, server.accessToken, 0, 10, '-createdAt') | |
355 | .then(res => { | |
356 | const jobs = res.body.data | |
357 | ||
358 | for (const job of jobs) { | |
359 | if (job.state === 'pending' || job.state === 'processing') { | |
360 | pendingRequests = true | |
361 | } | |
362 | } | |
363 | }) | |
364 | ||
365 | tasks.push(p) | |
366 | } | |
367 | ||
368 | await Promise.all(tasks) | |
369 | ||
370 | return pendingRequests | |
371 | } |