]>
Commit | Line | Data |
---|---|---|
68e70a74 C |
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | ||
3 | import 'mocha' | |
4 | import * as chai from 'chai' | |
5 | import { FfmpegCommand } from 'fluent-ffmpeg' | |
4ec52d04 | 6 | import { checkLiveCleanup } from '@server/tests/shared' |
c55e3d72 | 7 | import { wait } from '@shared/core-utils' |
26e3e98f | 8 | import { HttpStatusCode, LiveVideoCreate, LiveVideoError, VideoPrivacy, VideoState } from '@shared/models' |
68e70a74 | 9 | import { |
68e70a74 | 10 | cleanupTests, |
65e6e260 | 11 | ConfigCommand, |
254d3579 | 12 | createMultipleServers, |
4c7e60bc | 13 | doubleFollow, |
4ec52d04 | 14 | findExternalSavedVideo, |
cdd83816 | 15 | makeRawRequest, |
254d3579 | 16 | PeerTubeServer, |
68e70a74 C |
17 | setAccessTokensToServers, |
18 | setDefaultVideoChannel, | |
19 | stopFfmpeg, | |
20 | testFfmpegStreamError, | |
0305db28 JB |
21 | waitJobs, |
22 | waitUntilLivePublishedOnAllServers, | |
4ec52d04 C |
23 | waitUntilLiveReplacedByReplayOnAllServers, |
24 | waitUntilLiveWaitingOnAllServers | |
bf54587a | 25 | } from '@shared/server-commands' |
68e70a74 C |
26 | |
27 | const expect = chai.expect | |
28 | ||
29 | describe('Save replay setting', function () { | |
254d3579 | 30 | let servers: PeerTubeServer[] = [] |
68e70a74 C |
31 | let liveVideoUUID: string |
32 | let ffmpegCommand: FfmpegCommand | |
33 | ||
4ec52d04 | 34 | async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) { |
68e70a74 C |
35 | if (liveVideoUUID) { |
36 | try { | |
89d241a7 | 37 | await servers[0].videos.remove({ id: liveVideoUUID }) |
68e70a74 C |
38 | await waitJobs(servers) |
39 | } catch {} | |
40 | } | |
41 | ||
42 | const attributes: LiveVideoCreate = { | |
89d241a7 | 43 | channelId: servers[0].store.channel.id, |
68e70a74 C |
44 | privacy: VideoPrivacy.PUBLIC, |
45 | name: 'my super live', | |
4ec52d04 C |
46 | saveReplay: options.replay, |
47 | permanentLive: options.permanent | |
68e70a74 C |
48 | } |
49 | ||
89d241a7 | 50 | const { uuid } = await servers[0].live.create({ fields: attributes }) |
4f219914 | 51 | return uuid |
68e70a74 C |
52 | } |
53 | ||
98ebfa39 C |
54 | async function publishLive (options: { permanent: boolean, replay: boolean }) { |
55 | liveVideoUUID = await createLiveWrapper(options) | |
56 | ||
57 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | |
58 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) | |
59 | ||
60 | const liveDetails = await servers[0].videos.get({ id: liveVideoUUID }) | |
61 | ||
62 | await waitJobs(servers) | |
63 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | |
64 | ||
65 | return { ffmpegCommand, liveDetails } | |
66 | } | |
67 | ||
68 | async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean }) { | |
69 | const { ffmpegCommand, liveDetails } = await publishLive(options) | |
70 | ||
71 | await Promise.all([ | |
72 | servers[0].videos.remove({ id: liveVideoUUID }), | |
73 | testFfmpegStreamError(ffmpegCommand, true) | |
74 | ]) | |
75 | ||
76 | await waitJobs(servers) | |
77 | await wait(5000) | |
78 | await waitJobs(servers) | |
79 | ||
80 | return { liveDetails } | |
81 | } | |
82 | ||
83 | async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean }) { | |
84 | const { ffmpegCommand, liveDetails } = await publishLive(options) | |
85 | ||
86 | await Promise.all([ | |
87 | servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }), | |
88 | testFfmpegStreamError(ffmpegCommand, true) | |
89 | ]) | |
90 | ||
91 | await waitJobs(servers) | |
92 | await wait(5000) | |
93 | await waitJobs(servers) | |
94 | ||
95 | return { liveDetails } | |
96 | } | |
97 | ||
d23dd9fb | 98 | async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) { |
68e70a74 C |
99 | for (const server of servers) { |
100 | const length = existsInList ? 1 : 0 | |
101 | ||
89d241a7 | 102 | const { data, total } = await server.videos.list() |
d23dd9fb C |
103 | expect(data).to.have.lengthOf(length) |
104 | expect(total).to.equal(length) | |
68e70a74 | 105 | |
d23dd9fb | 106 | if (expectedStatus) { |
89d241a7 | 107 | await server.videos.get({ id: videoId, expectedStatus }) |
68e70a74 C |
108 | } |
109 | } | |
110 | } | |
111 | ||
112 | async function checkVideoState (videoId: string, state: VideoState) { | |
113 | for (const server of servers) { | |
89d241a7 | 114 | const video = await server.videos.get({ id: videoId }) |
d23dd9fb | 115 | expect(video.state.id).to.equal(state) |
68e70a74 C |
116 | } |
117 | } | |
118 | ||
119 | before(async function () { | |
120 | this.timeout(120000) | |
121 | ||
254d3579 | 122 | servers = await createMultipleServers(2) |
68e70a74 C |
123 | |
124 | // Get the access tokens | |
125 | await setAccessTokensToServers(servers) | |
126 | await setDefaultVideoChannel(servers) | |
127 | ||
128 | // Server 1 and server 2 follow each other | |
129 | await doubleFollow(servers[0], servers[1]) | |
130 | ||
89d241a7 | 131 | await servers[0].config.updateCustomSubConfig({ |
65e6e260 C |
132 | newConfig: { |
133 | live: { | |
134 | enabled: true, | |
135 | allowReplay: true, | |
136 | maxDuration: -1, | |
137 | transcoding: { | |
138 | enabled: false, | |
139 | resolutions: ConfigCommand.getCustomConfigResolutions(true) | |
140 | } | |
68e70a74 C |
141 | } |
142 | } | |
143 | }) | |
144 | }) | |
145 | ||
146 | describe('With save replay disabled', function () { | |
26e3e98f C |
147 | let sessionStartDateMin: Date |
148 | let sessionStartDateMax: Date | |
149 | let sessionEndDateMin: Date | |
68e70a74 | 150 | |
68e70a74 C |
151 | it('Should correctly create and federate the "waiting for stream" live', async function () { |
152 | this.timeout(20000) | |
153 | ||
4ec52d04 | 154 | liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false }) |
68e70a74 C |
155 | |
156 | await waitJobs(servers) | |
157 | ||
f2eb23cd | 158 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) |
68e70a74 C |
159 | await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) |
160 | }) | |
161 | ||
162 | it('Should correctly have updated the live and federated it when streaming in the live', async function () { | |
fae6e4da | 163 | this.timeout(30000) |
68e70a74 | 164 | |
89d241a7 | 165 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
fae6e4da | 166 | |
26e3e98f | 167 | sessionStartDateMin = new Date() |
0305db28 | 168 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
26e3e98f | 169 | sessionStartDateMax = new Date() |
68e70a74 C |
170 | |
171 | await waitJobs(servers) | |
172 | ||
f2eb23cd | 173 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
68e70a74 C |
174 | await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) |
175 | }) | |
176 | ||
177 | it('Should correctly delete the video files after the stream ended', async function () { | |
59fd824c | 178 | this.timeout(40000) |
68e70a74 | 179 | |
26e3e98f | 180 | sessionEndDateMin = new Date() |
68e70a74 C |
181 | await stopFfmpeg(ffmpegCommand) |
182 | ||
fae6e4da | 183 | for (const server of servers) { |
89d241a7 | 184 | await server.live.waitUntilEnded({ videoId: liveVideoUUID }) |
fae6e4da | 185 | } |
68e70a74 C |
186 | await waitJobs(servers) |
187 | ||
188 | // Live still exist, but cannot be played anymore | |
f2eb23cd | 189 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) |
68e70a74 C |
190 | await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) |
191 | ||
192 | // No resolutions saved since we did not save replay | |
4ec52d04 | 193 | await checkLiveCleanup(servers[0], liveVideoUUID, []) |
68e70a74 C |
194 | }) |
195 | ||
26e3e98f C |
196 | it('Should have appropriate ended session', async function () { |
197 | const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) | |
198 | expect(total).to.equal(1) | |
199 | expect(data).to.have.lengthOf(1) | |
200 | ||
201 | const session = data[0] | |
202 | ||
203 | const startDate = new Date(session.startDate) | |
204 | expect(startDate).to.be.above(sessionStartDateMin) | |
205 | expect(startDate).to.be.below(sessionStartDateMax) | |
206 | ||
207 | expect(session.endDate).to.exist | |
208 | expect(new Date(session.endDate)).to.be.above(sessionEndDateMin) | |
209 | ||
210 | expect(session.error).to.not.exist | |
211 | expect(session.replayVideo).to.not.exist | |
212 | }) | |
213 | ||
68e70a74 C |
214 | it('Should correctly terminate the stream on blacklist and delete the live', async function () { |
215 | this.timeout(40000) | |
216 | ||
98ebfa39 | 217 | await publishLiveAndBlacklist({ permanent: false, replay: false }) |
68e70a74 C |
218 | |
219 | await checkVideosExist(liveVideoUUID, false) | |
220 | ||
89d241a7 C |
221 | await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) |
222 | await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | |
68e70a74 | 223 | |
94d721ef C |
224 | await wait(5000) |
225 | await waitJobs(servers) | |
4ec52d04 | 226 | await checkLiveCleanup(servers[0], liveVideoUUID, []) |
68e70a74 C |
227 | }) |
228 | ||
26e3e98f C |
229 | it('Should have blacklisted session error', async function () { |
230 | const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID }) | |
231 | expect(session.startDate).to.exist | |
232 | expect(session.endDate).to.exist | |
233 | ||
234 | expect(session.error).to.equal(LiveVideoError.BLACKLISTED) | |
235 | expect(session.replayVideo).to.not.exist | |
236 | }) | |
237 | ||
68e70a74 C |
238 | it('Should correctly terminate the stream on delete and delete the video', async function () { |
239 | this.timeout(40000) | |
240 | ||
98ebfa39 | 241 | await publishLiveAndDelete({ permanent: false, replay: false }) |
68e70a74 | 242 | |
f2eb23cd | 243 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) |
4ec52d04 | 244 | await checkLiveCleanup(servers[0], liveVideoUUID, []) |
68e70a74 C |
245 | }) |
246 | }) | |
247 | ||
4ec52d04 | 248 | describe('With save replay enabled on non permanent live', function () { |
68e70a74 C |
249 | |
250 | it('Should correctly create and federate the "waiting for stream" live', async function () { | |
251 | this.timeout(20000) | |
252 | ||
4ec52d04 | 253 | liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true }) |
68e70a74 C |
254 | |
255 | await waitJobs(servers) | |
256 | ||
f2eb23cd | 257 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) |
68e70a74 C |
258 | await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) |
259 | }) | |
260 | ||
261 | it('Should correctly have updated the live and federated it when streaming in the live', async function () { | |
262 | this.timeout(20000) | |
263 | ||
89d241a7 | 264 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
0305db28 | 265 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
68e70a74 C |
266 | |
267 | await waitJobs(servers) | |
268 | ||
f2eb23cd | 269 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
68e70a74 C |
270 | await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) |
271 | }) | |
272 | ||
273 | it('Should correctly have saved the live and federated it after the streaming', async function () { | |
274 | this.timeout(30000) | |
275 | ||
276 | await stopFfmpeg(ffmpegCommand) | |
277 | ||
4ec52d04 | 278 | await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID) |
68e70a74 C |
279 | await waitJobs(servers) |
280 | ||
281 | // Live has been transcoded | |
f2eb23cd | 282 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
68e70a74 C |
283 | await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) |
284 | }) | |
285 | ||
26e3e98f C |
286 | it('Should find the replay live session', async function () { |
287 | const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID }) | |
288 | ||
289 | expect(session).to.exist | |
290 | ||
291 | expect(session.startDate).to.exist | |
292 | expect(session.endDate).to.exist | |
293 | ||
294 | expect(session.error).to.not.exist | |
295 | ||
296 | expect(session.replayVideo).to.exist | |
297 | expect(session.replayVideo.id).to.exist | |
298 | expect(session.replayVideo.shortUUID).to.exist | |
299 | expect(session.replayVideo.uuid).to.equal(liveVideoUUID) | |
300 | }) | |
301 | ||
68e70a74 C |
302 | it('Should update the saved live and correctly federate the updated attributes', async function () { |
303 | this.timeout(30000) | |
304 | ||
89d241a7 | 305 | await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated' } }) |
68e70a74 C |
306 | await waitJobs(servers) |
307 | ||
308 | for (const server of servers) { | |
89d241a7 | 309 | const video = await server.videos.get({ id: liveVideoUUID }) |
d23dd9fb C |
310 | expect(video.name).to.equal('video updated') |
311 | expect(video.isLive).to.be.false | |
68e70a74 C |
312 | } |
313 | }) | |
314 | ||
315 | it('Should have cleaned up the live files', async function () { | |
4ec52d04 | 316 | await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) |
68e70a74 C |
317 | }) |
318 | ||
319 | it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { | |
320 | this.timeout(40000) | |
321 | ||
98ebfa39 | 322 | await publishLiveAndBlacklist({ permanent: false, replay: true }) |
68e70a74 C |
323 | |
324 | await checkVideosExist(liveVideoUUID, false) | |
325 | ||
89d241a7 C |
326 | await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) |
327 | await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | |
68e70a74 | 328 | |
94d721ef C |
329 | await wait(5000) |
330 | await waitJobs(servers) | |
4ec52d04 | 331 | await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) |
68e70a74 C |
332 | }) |
333 | ||
334 | it('Should correctly terminate the stream on delete and delete the video', async function () { | |
335 | this.timeout(40000) | |
336 | ||
98ebfa39 | 337 | await publishLiveAndDelete({ permanent: false, replay: true }) |
68e70a74 | 338 | |
f2eb23cd | 339 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) |
4ec52d04 C |
340 | await checkLiveCleanup(servers[0], liveVideoUUID, []) |
341 | }) | |
342 | }) | |
343 | ||
344 | describe('With save replay enabled on permanent live', function () { | |
345 | let lastReplayUUID: string | |
346 | ||
347 | it('Should correctly create and federate the "waiting for stream" live', async function () { | |
348 | this.timeout(20000) | |
349 | ||
350 | liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true }) | |
351 | ||
352 | await waitJobs(servers) | |
353 | ||
354 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) | |
355 | await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) | |
356 | }) | |
357 | ||
358 | it('Should correctly have updated the live and federated it when streaming in the live', async function () { | |
359 | this.timeout(20000) | |
360 | ||
361 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | |
362 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) | |
363 | ||
364 | await waitJobs(servers) | |
365 | ||
366 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | |
367 | await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) | |
368 | }) | |
369 | ||
370 | it('Should correctly have saved the live and federated it after the streaming', async function () { | |
371 | this.timeout(30000) | |
372 | ||
373 | const liveDetails = await servers[0].videos.get({ id: liveVideoUUID }) | |
374 | ||
375 | await stopFfmpeg(ffmpegCommand) | |
376 | ||
377 | await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID) | |
378 | await waitJobs(servers) | |
379 | ||
380 | const video = await findExternalSavedVideo(servers[0], liveDetails) | |
381 | expect(video).to.exist | |
382 | ||
383 | for (const server of servers) { | |
384 | await server.videos.get({ id: video.uuid }) | |
385 | } | |
386 | ||
387 | lastReplayUUID = video.uuid | |
388 | }) | |
389 | ||
26e3e98f C |
390 | it('Should have appropriate ended session and replay live session', async function () { |
391 | const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) | |
392 | expect(total).to.equal(1) | |
393 | expect(data).to.have.lengthOf(1) | |
394 | ||
395 | const sessionFromLive = data[0] | |
396 | const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID }) | |
397 | ||
398 | for (const session of [ sessionFromLive, sessionFromReplay ]) { | |
399 | expect(session.startDate).to.exist | |
400 | expect(session.endDate).to.exist | |
401 | ||
402 | expect(session.error).to.not.exist | |
403 | ||
404 | expect(session.replayVideo).to.exist | |
405 | expect(session.replayVideo.id).to.exist | |
406 | expect(session.replayVideo.shortUUID).to.exist | |
407 | expect(session.replayVideo.uuid).to.equal(lastReplayUUID) | |
408 | } | |
409 | }) | |
410 | ||
4ec52d04 C |
411 | it('Should have cleaned up the live files', async function () { |
412 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | |
413 | }) | |
414 | ||
415 | it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { | |
416 | this.timeout(60000) | |
417 | ||
418 | await servers[0].videos.remove({ id: lastReplayUUID }) | |
98ebfa39 | 419 | const { liveDetails } = await publishLiveAndBlacklist({ permanent: true, replay: true }) |
4ec52d04 C |
420 | |
421 | const replay = await findExternalSavedVideo(servers[0], liveDetails) | |
422 | expect(replay).to.exist | |
423 | ||
424 | for (const videoId of [ liveVideoUUID, replay.uuid ]) { | |
425 | await checkVideosExist(videoId, false) | |
426 | ||
427 | await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) | |
428 | await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | |
429 | } | |
430 | ||
431 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | |
432 | }) | |
433 | ||
434 | it('Should correctly terminate the stream on delete and not save the video', async function () { | |
435 | this.timeout(40000) | |
436 | ||
98ebfa39 | 437 | const { liveDetails } = await publishLiveAndDelete({ permanent: true, replay: true }) |
4ec52d04 C |
438 | |
439 | const replay = await findExternalSavedVideo(servers[0], liveDetails) | |
440 | expect(replay).to.not.exist | |
441 | ||
442 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) | |
443 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | |
68e70a74 | 444 | }) |
5333788c C |
445 | |
446 | it('Should correctly save replays with multiple sessions', async function () { | |
447 | this.timeout(120000) | |
448 | ||
449 | liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true }) | |
450 | await waitJobs(servers) | |
451 | ||
452 | // Streaming session #1 | |
453 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | |
454 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) | |
455 | await stopFfmpeg(ffmpegCommand) | |
456 | await servers[0].live.waitUntilWaiting({ videoId: liveVideoUUID }) | |
457 | ||
458 | // Streaming session #2 | |
459 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | |
460 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) | |
cdd83816 C |
461 | |
462 | await wait(5000) | |
463 | const video = await servers[0].videos.get({ id: liveVideoUUID }) | |
464 | expect(video.streamingPlaylists).to.have.lengthOf(1) | |
465 | await makeRawRequest(video.streamingPlaylists[0].playlistUrl) | |
466 | ||
5333788c C |
467 | await stopFfmpeg(ffmpegCommand) |
468 | await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID) | |
469 | ||
470 | // Wait for replays | |
471 | await waitJobs(servers) | |
472 | ||
473 | const { total, data: sessions } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) | |
474 | ||
475 | expect(total).to.equal(2) | |
476 | expect(sessions).to.have.lengthOf(2) | |
477 | ||
478 | for (const session of sessions) { | |
479 | expect(session.error).to.be.null | |
480 | expect(session.replayVideo).to.exist | |
481 | ||
482 | await servers[0].videos.get({ id: session.replayVideo.uuid }) | |
483 | } | |
484 | }) | |
68e70a74 C |
485 | }) |
486 | ||
487 | after(async function () { | |
488 | await cleanupTests(servers) | |
489 | }) | |
490 | }) |