aboutsummaryrefslogtreecommitdiffhomepage
path: root/shared/extra-utils/server/servers.ts
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-07-21 15:51:30 +0200
committerChocobozzz <me@florianbigard.com>2021-07-21 15:51:30 +0200
commita24bd1ed41b43790bab6ba789580bb4e85f07d85 (patch)
treea54b0f6c921ba83a6e909cd0ced325b2d4b8863c /shared/extra-utils/server/servers.ts
parent5f26f13b3c16ac5ae0a3b0a7142d84a9528cf565 (diff)
parentc63830f15403ac4e750829f27d8bbbdc9a59282c (diff)
downloadPeerTube-a24bd1ed41b43790bab6ba789580bb4e85f07d85.tar.gz
PeerTube-a24bd1ed41b43790bab6ba789580bb4e85f07d85.tar.zst
PeerTube-a24bd1ed41b43790bab6ba789580bb4e85f07d85.zip
Merge branch 'next' into develop
Diffstat (limited to 'shared/extra-utils/server/servers.ts')
-rw-r--r--shared/extra-utils/server/servers.ts375
1 files changed, 20 insertions, 355 deletions
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts
index 28e431e94..87d7e9449 100644
--- a/shared/extra-utils/server/servers.ts
+++ b/shared/extra-utils/server/servers.ts
@@ -1,384 +1,49 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ 1import { ensureDir } from 'fs-extra'
2import { isGithubCI } from '../miscs'
3import { PeerTubeServer, RunServerOptions } from './server'
2 4
3import { expect } from 'chai' 5async function createSingleServer (serverNumber: number, configOverride?: Object, args = [], options: RunServerOptions = {}) {
4import { ChildProcess, exec, fork } from 'child_process' 6 const server = new PeerTubeServer({ serverNumber })
5import { copy, ensureDir, pathExists, readdir, readFile, remove } from 'fs-extra'
6import { join } from 'path'
7import { randomInt } from '../../core-utils/miscs/miscs'
8import { VideoChannel } from '../../models/videos'
9import { buildServerDirectory, getFileSize, isGithubCI, root, wait } from '../miscs/miscs'
10import { makeGetRequest } from '../requests/requests'
11 7
12interface ServerInfo { 8 await server.flushAndRun(configOverride, args, options)
13 app: ChildProcess
14
15 url: string
16 host: string
17 hostname: string
18 port: number
19
20 rtmpPort: number
21
22 parallel: boolean
23 internalServerNumber: number
24 serverNumber: number
25
26 client: {
27 id: string
28 secret: string
29 }
30
31 user: {
32 username: string
33 password: string
34 email?: string
35 }
36
37 customConfigFile?: string
38
39 accessToken?: string
40 refreshToken?: string
41 videoChannel?: VideoChannel
42
43 video?: {
44 id: number
45 uuid: string
46 shortUUID: string
47 name?: string
48 url?: string
49
50 account?: {
51 name: string
52 }
53
54 embedPath?: string
55 }
56
57 remoteVideo?: {
58 id: number
59 uuid: string
60 }
61
62 videos?: { id: number, uuid: string }[]
63}
64
65function parallelTests () {
66 return process.env.MOCHA_PARALLEL === 'true'
67}
68
69function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) {
70 const apps = []
71 let i = 0
72
73 return new Promise<ServerInfo[]>(res => {
74 function anotherServerDone (serverNumber, app) {
75 apps[serverNumber - 1] = app
76 i++
77 if (i === totalServers) {
78 return res(apps)
79 }
80 }
81
82 for (let j = 1; j <= totalServers; j++) {
83 flushAndRunServer(j, configOverride).then(app => anotherServerDone(j, app))
84 }
85 })
86}
87
88function flushTests (serverNumber?: number) {
89 return new Promise<void>((res, rej) => {
90 const suffix = serverNumber ? ` -- ${serverNumber}` : ''
91
92 return exec('npm run clean:server:test' + suffix, (err, _stdout, stderr) => {
93 if (err || stderr) return rej(err || new Error(stderr))
94
95 return res()
96 })
97 })
98}
99
100function randomServer () {
101 const low = 10
102 const high = 10000
103
104 return randomInt(low, high)
105}
106
107function randomRTMP () {
108 const low = 1900
109 const high = 2100
110
111 return randomInt(low, high)
112}
113
114type RunServerOptions = {
115 hideLogs?: boolean
116 execArgv?: string[]
117}
118
119async function flushAndRunServer (serverNumber: number, configOverride?: Object, args = [], options: RunServerOptions = {}) {
120 const parallel = parallelTests()
121
122 const internalServerNumber = parallel ? randomServer() : serverNumber
123 const rtmpPort = parallel ? randomRTMP() : 1936
124 const port = 9000 + internalServerNumber
125
126 await flushTests(internalServerNumber)
127
128 const server: ServerInfo = {
129 app: null,
130 port,
131 internalServerNumber,
132 rtmpPort,
133 parallel,
134 serverNumber,
135 url: `http://localhost:${port}`,
136 host: `localhost:${port}`,
137 hostname: 'localhost',
138 client: {
139 id: null,
140 secret: null
141 },
142 user: {
143 username: null,
144 password: null
145 }
146 }
147
148 return runServer(server, configOverride, args, options)
149}
150
151async function runServer (server: ServerInfo, configOverrideArg?: any, args = [], options: RunServerOptions = {}) {
152 // These actions are async so we need to be sure that they have both been done
153 const serverRunString = {
154 'HTTP server listening': false
155 }
156 const key = 'Database peertube_test' + server.internalServerNumber + ' is ready'
157 serverRunString[key] = false
158
159 const regexps = {
160 client_id: 'Client id: (.+)',
161 client_secret: 'Client secret: (.+)',
162 user_username: 'Username: (.+)',
163 user_password: 'User password: (.+)'
164 }
165
166 if (server.internalServerNumber !== server.serverNumber) {
167 const basePath = join(root(), 'config')
168
169 const tmpConfigFile = join(basePath, `test-${server.internalServerNumber}.yaml`)
170 await copy(join(basePath, `test-${server.serverNumber}.yaml`), tmpConfigFile)
171
172 server.customConfigFile = tmpConfigFile
173 }
174
175 const configOverride: any = {}
176
177 if (server.parallel) {
178 Object.assign(configOverride, {
179 listen: {
180 port: server.port
181 },
182 webserver: {
183 port: server.port
184 },
185 database: {
186 suffix: '_test' + server.internalServerNumber
187 },
188 storage: {
189 tmp: `test${server.internalServerNumber}/tmp/`,
190 avatars: `test${server.internalServerNumber}/avatars/`,
191 videos: `test${server.internalServerNumber}/videos/`,
192 streaming_playlists: `test${server.internalServerNumber}/streaming-playlists/`,
193 redundancy: `test${server.internalServerNumber}/redundancy/`,
194 logs: `test${server.internalServerNumber}/logs/`,
195 previews: `test${server.internalServerNumber}/previews/`,
196 thumbnails: `test${server.internalServerNumber}/thumbnails/`,
197 torrents: `test${server.internalServerNumber}/torrents/`,
198 captions: `test${server.internalServerNumber}/captions/`,
199 cache: `test${server.internalServerNumber}/cache/`,
200 plugins: `test${server.internalServerNumber}/plugins/`
201 },
202 admin: {
203 email: `admin${server.internalServerNumber}@example.com`
204 },
205 live: {
206 rtmp: {
207 port: server.rtmpPort
208 }
209 }
210 })
211 }
212
213 if (configOverrideArg !== undefined) {
214 Object.assign(configOverride, configOverrideArg)
215 }
216
217 // Share the environment
218 const env = Object.create(process.env)
219 env['NODE_ENV'] = 'test'
220 env['NODE_APP_INSTANCE'] = server.internalServerNumber.toString()
221 env['NODE_CONFIG'] = JSON.stringify(configOverride)
222
223 const forkOptions = {
224 silent: true,
225 env,
226 detached: true,
227 execArgv: options.execArgv || []
228 }
229
230 return new Promise<ServerInfo>(res => {
231 server.app = fork(join(root(), 'dist', 'server.js'), args, forkOptions)
232 server.app.stdout.on('data', function onStdout (data) {
233 let dontContinue = false
234
235 // Capture things if we want to
236 for (const key of Object.keys(regexps)) {
237 const regexp = regexps[key]
238 const matches = data.toString().match(regexp)
239 if (matches !== null) {
240 if (key === 'client_id') server.client.id = matches[1]
241 else if (key === 'client_secret') server.client.secret = matches[1]
242 else if (key === 'user_username') server.user.username = matches[1]
243 else if (key === 'user_password') server.user.password = matches[1]
244 }
245 }
246
247 // Check if all required sentences are here
248 for (const key of Object.keys(serverRunString)) {
249 if (data.toString().indexOf(key) !== -1) serverRunString[key] = true
250 if (serverRunString[key] === false) dontContinue = true
251 }
252
253 // If no, there is maybe one thing not already initialized (client/user credentials generation...)
254 if (dontContinue === true) return
255
256 if (options.hideLogs === false) {
257 console.log(data.toString())
258 } else {
259 server.app.stdout.removeListener('data', onStdout)
260 }
261
262 process.on('exit', () => {
263 try {
264 process.kill(server.app.pid)
265 } catch { /* empty */ }
266 })
267
268 res(server)
269 })
270 })
271}
272
273async function reRunServer (server: ServerInfo, configOverride?: any) {
274 const newServer = await runServer(server, configOverride)
275 server.app = newServer.app
276 9
277 return server 10 return server
278} 11}
279 12
280async function checkTmpIsEmpty (server: ServerInfo) { 13function createMultipleServers (totalServers: number, configOverride?: Object) {
281 await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) 14 const serverPromises: Promise<PeerTubeServer>[] = []
282 15
283 if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) { 16 for (let i = 1; i <= totalServers; i++) {
284 await checkDirectoryIsEmpty(server, 'tmp/hls') 17 serverPromises.push(createSingleServer(i, configOverride))
285 } 18 }
286}
287
288async function checkDirectoryIsEmpty (server: ServerInfo, directory: string, exceptions: string[] = []) {
289 const testDirectory = 'test' + server.internalServerNumber
290
291 const directoryPath = join(root(), testDirectory, directory)
292 19
293 const directoryExists = await pathExists(directoryPath) 20 return Promise.all(serverPromises)
294 expect(directoryExists).to.be.true
295
296 const files = await readdir(directoryPath)
297 const filtered = files.filter(f => exceptions.includes(f) === false)
298
299 expect(filtered).to.have.lengthOf(0)
300} 21}
301 22
302function killallServers (servers: ServerInfo[]) { 23async function killallServers (servers: PeerTubeServer[]) {
303 for (const server of servers) { 24 return Promise.all(servers.map(s => s.kill()))
304 if (!server.app) continue
305
306 process.kill(-server.app.pid)
307 server.app = null
308 }
309} 25}
310 26
311async function cleanupTests (servers: ServerInfo[]) { 27async function cleanupTests (servers: PeerTubeServer[]) {
312 killallServers(servers) 28 await killallServers(servers)
313 29
314 if (isGithubCI()) { 30 if (isGithubCI()) {
315 await ensureDir('artifacts') 31 await ensureDir('artifacts')
316 } 32 }
317 33
318 const p: Promise<any>[] = [] 34 let p: Promise<any>[] = []
319 for (const server of servers) { 35 for (const server of servers) {
320 if (isGithubCI()) { 36 p = p.concat(server.servers.cleanupTests())
321 const origin = await buildServerDirectory(server, 'logs/peertube.log')
322 const destname = `peertube-${server.internalServerNumber}.log`
323 console.log('Saving logs %s.', destname)
324
325 await copy(origin, join('artifacts', destname))
326 }
327
328 if (server.parallel) {
329 p.push(flushTests(server.internalServerNumber))
330 }
331
332 if (server.customConfigFile) {
333 p.push(remove(server.customConfigFile))
334 }
335 } 37 }
336 38
337 return Promise.all(p) 39 return Promise.all(p)
338} 40}
339 41
340async function waitUntilLog (server: ServerInfo, str: string, count = 1, strictCount = true) {
341 const logfile = buildServerDirectory(server, 'logs/peertube.log')
342
343 while (true) {
344 const buf = await readFile(logfile)
345
346 const matches = buf.toString().match(new RegExp(str, 'g'))
347 if (matches && matches.length === count) return
348 if (matches && strictCount === false && matches.length >= count) return
349
350 await wait(1000)
351 }
352}
353
354async function getServerFileSize (server: ServerInfo, subPath: string) {
355 const path = buildServerDirectory(server, subPath)
356
357 return getFileSize(path)
358}
359
360function makePingRequest (server: ServerInfo) {
361 return makeGetRequest({
362 url: server.url,
363 path: '/api/v1/ping',
364 statusCodeExpected: 200
365 })
366}
367
368// --------------------------------------------------------------------------- 42// ---------------------------------------------------------------------------
369 43
370export { 44export {
371 checkDirectoryIsEmpty, 45 createSingleServer,
372 checkTmpIsEmpty, 46 createMultipleServers,
373 getServerFileSize,
374 ServerInfo,
375 parallelTests,
376 cleanupTests, 47 cleanupTests,
377 flushAndRunMultipleServers, 48 killallServers
378 flushTests,
379 makePingRequest,
380 flushAndRunServer,
381 killallServers,
382 reRunServer,
383 waitUntilLog
384} 49}