1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
3 import { expect } from 'chai'
4 import { wait } from '@shared/core-utils'
11 RunnerJobVODWebVideoTranscodingPayload,
12 RunnerRegistrationToken
13 } from '@shared/models'
18 setAccessTokensToServers,
19 setDefaultVideoChannel,
21 } from '@shared/server-commands'
23 describe('Test runner common actions', function () {
24 let server: PeerTubeServer
25 let registrationToken: string
26 let runnerToken: string
27 let jobMaxPriority: string
29 before(async function () {
32 server = await createSingleServer(1, {
40 await setAccessTokensToServers([ server ])
41 await setDefaultVideoChannel([ server ])
43 await server.config.enableTranscoding(true, true)
44 await server.config.enableRemoteTranscoding()
47 describe('Managing runner registration tokens', function () {
48 let base: RunnerRegistrationToken[]
49 let registrationTokenToDelete: RunnerRegistrationToken
51 it('Should have a default registration token', async function () {
52 const { total, data } = await server.runnerRegistrationTokens.list()
54 expect(total).to.equal(1)
55 expect(data).to.have.lengthOf(1)
58 expect(token.id).to.exist
59 expect(token.createdAt).to.exist
60 expect(token.updatedAt).to.exist
61 expect(token.registeredRunnersCount).to.equal(0)
62 expect(token.registrationToken).to.exist
65 it('Should create other registration tokens', async function () {
66 await server.runnerRegistrationTokens.generate()
67 await server.runnerRegistrationTokens.generate()
69 const { total, data } = await server.runnerRegistrationTokens.list()
70 expect(total).to.equal(3)
71 expect(data).to.have.lengthOf(3)
74 it('Should list registration tokens', async function () {
76 const { total, data } = await server.runnerRegistrationTokens.list({ sort: 'createdAt' })
77 expect(total).to.equal(3)
78 expect(data).to.have.lengthOf(3)
79 expect(new Date(data[0].createdAt)).to.be.below(new Date(data[1].createdAt))
80 expect(new Date(data[1].createdAt)).to.be.below(new Date(data[2].createdAt))
84 registrationTokenToDelete = data[0]
85 registrationToken = data[1].registrationToken
89 const { total, data } = await server.runnerRegistrationTokens.list({ sort: '-createdAt', start: 2, count: 1 })
90 expect(total).to.equal(3)
91 expect(data).to.have.lengthOf(1)
92 expect(data[0].registrationToken).to.equal(base[0].registrationToken)
96 it('Should have appropriate registeredRunnersCount for registration tokens', async function () {
97 await server.runners.register({ name: 'to delete 1', registrationToken: registrationTokenToDelete.registrationToken })
98 await server.runners.register({ name: 'to delete 2', registrationToken: registrationTokenToDelete.registrationToken })
100 const { data } = await server.runnerRegistrationTokens.list()
102 for (const d of data) {
103 if (d.registrationToken === registrationTokenToDelete.registrationToken) {
104 expect(d.registeredRunnersCount).to.equal(2)
106 expect(d.registeredRunnersCount).to.equal(0)
110 const { data: runners } = await server.runners.list()
111 expect(runners).to.have.lengthOf(2)
114 it('Should delete a registration token', async function () {
115 await server.runnerRegistrationTokens.delete({ id: registrationTokenToDelete.id })
117 const { total, data } = await server.runnerRegistrationTokens.list({ sort: 'createdAt' })
118 expect(total).to.equal(2)
119 expect(data).to.have.lengthOf(2)
121 for (const d of data) {
122 expect(d.registeredRunnersCount).to.equal(0)
123 expect(d.registrationToken).to.not.equal(registrationTokenToDelete.registrationToken)
127 it('Should have removed runners of this registration token', async function () {
128 const { data: runners } = await server.runners.list()
129 expect(runners).to.have.lengthOf(0)
133 describe('Managing runners', function () {
136 it('Should not have runners available', async function () {
137 const { total, data } = await server.runners.list()
139 expect(data).to.have.lengthOf(0)
140 expect(total).to.equal(0)
143 it('Should register runners', async function () {
144 const now = new Date()
146 const result = await server.runners.register({
148 description: 'my super runner 1',
151 expect(result.runnerToken).to.exist
152 runnerToken = result.runnerToken
154 await server.runners.register({
159 const { total, data } = await server.runners.list({ sort: 'createdAt' })
160 expect(total).to.equal(2)
161 expect(data).to.have.lengthOf(2)
163 for (const d of data) {
164 expect(d.id).to.exist
165 expect(d.createdAt).to.exist
166 expect(d.updatedAt).to.exist
167 expect(new Date(d.createdAt)).to.be.above(now)
168 expect(new Date(d.updatedAt)).to.be.above(now)
169 expect(new Date(d.lastContact)).to.be.above(now)
170 expect(d.ip).to.exist
173 expect(data[0].name).to.equal('runner 1')
174 expect(data[0].description).to.equal('my super runner 1')
176 expect(data[1].name).to.equal('runner 2')
177 expect(data[1].description).to.be.null
182 it('Should list runners', async function () {
183 const { total, data } = await server.runners.list({ sort: '-createdAt', start: 1, count: 1 })
185 expect(total).to.equal(2)
186 expect(data).to.have.lengthOf(1)
187 expect(data[0].name).to.equal('runner 1')
190 it('Should delete a runner', async function () {
191 await server.runners.delete({ id: toDelete.id })
193 const { total, data } = await server.runners.list()
195 expect(total).to.equal(1)
196 expect(data).to.have.lengthOf(1)
197 expect(data[0].name).to.equal('runner 1')
200 it('Should unregister a runner', async function () {
201 const registered = await server.runners.autoRegisterRunner()
204 const { total, data } = await server.runners.list()
205 expect(total).to.equal(2)
206 expect(data).to.have.lengthOf(2)
209 await server.runners.unregister({ runnerToken: registered })
212 const { total, data } = await server.runners.list()
213 expect(total).to.equal(1)
214 expect(data).to.have.lengthOf(1)
215 expect(data[0].name).to.equal('runner 1')
220 describe('Managing runner jobs', function () {
223 let lastRunnerContact: Date
224 let failedJob: RunnerJob
226 async function checkMainJobState (
227 mainJobState: RunnerJobState,
228 otherJobStates: RunnerJobState[] = [ RunnerJobState.PENDING, RunnerJobState.WAITING_FOR_PARENT_JOB ]
230 const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
232 for (const job of data) {
233 if (job.uuid === jobUUID) {
234 expect(job.state.id).to.equal(mainJobState)
236 expect(otherJobStates).to.include(job.state.id)
241 function getMainJob () {
242 return server.runnerJobs.getJob({ uuid: jobUUID })
245 describe('List jobs', function () {
247 it('Should not have jobs', async function () {
248 const { total, data } = await server.runnerJobs.list()
250 expect(data).to.have.lengthOf(0)
251 expect(total).to.equal(0)
254 it('Should upload a video and have available jobs', async function () {
255 await server.videos.quickUpload({ name: 'to transcode' })
256 await waitJobs([ server ])
258 const { total, data } = await server.runnerJobs.list()
260 expect(data).to.have.lengthOf(10)
261 expect(total).to.equal(10)
263 for (const job of data) {
264 expect(job.startedAt).to.not.exist
265 expect(job.finishedAt).to.not.exist
266 expect(job.payload).to.exist
267 expect(job.privatePayload).to.exist
270 const hlsJobs = data.filter(d => d.type === 'vod-hls-transcoding')
271 const webVideoJobs = data.filter(d => d.type === 'vod-web-video-transcoding')
273 expect(hlsJobs).to.have.lengthOf(5)
274 expect(webVideoJobs).to.have.lengthOf(5)
276 const pendingJobs = data.filter(d => d.state.id === RunnerJobState.PENDING)
277 const waitingJobs = data.filter(d => d.state.id === RunnerJobState.WAITING_FOR_PARENT_JOB)
279 expect(pendingJobs).to.have.lengthOf(1)
280 expect(waitingJobs).to.have.lengthOf(9)
283 it('Should upload another video and list/sort jobs', async function () {
284 await server.videos.quickUpload({ name: 'to transcode 2' })
285 await waitJobs([ server ])
288 const { total, data } = await server.runnerJobs.list({ start: 0, count: 30 })
290 expect(data).to.have.lengthOf(20)
291 expect(total).to.equal(20)
293 jobUUID = data[16].uuid
297 const { total, data } = await server.runnerJobs.list({ start: 3, count: 1, sort: 'createdAt' })
298 expect(total).to.equal(20)
300 expect(data).to.have.lengthOf(1)
301 expect(data[0].uuid).to.equal(jobUUID)
305 let previousPriority = Infinity
306 const { total, data } = await server.runnerJobs.list({ start: 0, count: 100, sort: '-priority' })
307 expect(total).to.equal(20)
309 for (const job of data) {
310 expect(job.priority).to.be.at.most(previousPriority)
311 previousPriority = job.priority
313 if (job.state.id === RunnerJobState.PENDING) {
314 jobMaxPriority = job.uuid
320 it('Should search jobs', async function () {
322 const { total, data } = await server.runnerJobs.list({ search: jobUUID })
324 expect(data).to.have.lengthOf(1)
325 expect(total).to.equal(1)
327 expect(data[0].uuid).to.equal(jobUUID)
331 const { total, data } = await server.runnerJobs.list({ search: 'toto' })
333 expect(data).to.have.lengthOf(0)
334 expect(total).to.equal(0)
338 const { total, data } = await server.runnerJobs.list({ search: 'hls' })
340 expect(data).to.not.have.lengthOf(0)
341 expect(total).to.not.equal(0)
346 describe('Accept/update/abort/process a job', function () {
348 it('Should request available jobs', async function () {
349 lastRunnerContact = new Date()
351 const { availableJobs } = await server.runnerJobs.request({ runnerToken })
353 // Only optimize jobs are available
354 expect(availableJobs).to.have.lengthOf(2)
356 for (const job of availableJobs) {
357 expect(job.uuid).to.exist
358 expect(job.payload.input).to.exist
359 expect((job.payload as RunnerJobVODWebVideoTranscodingPayload).output).to.exist
361 expect((job as RunnerJobAdmin).privatePayload).to.not.exist
364 const hlsJobs = availableJobs.filter(d => d.type === 'vod-hls-transcoding')
365 const webVideoJobs = availableJobs.filter(d => d.type === 'vod-web-video-transcoding')
367 expect(hlsJobs).to.have.lengthOf(0)
368 expect(webVideoJobs).to.have.lengthOf(2)
370 jobUUID = webVideoJobs[0].uuid
373 it('Should have sorted available jobs by priority', async function () {
374 const { availableJobs } = await server.runnerJobs.request({ runnerToken })
376 expect(availableJobs[0].uuid).to.equal(jobMaxPriority)
379 it('Should have last runner contact updated', async function () {
382 const { data } = await server.runners.list({ sort: 'createdAt' })
383 expect(new Date(data[0].lastContact)).to.be.above(lastRunnerContact)
386 it('Should accept a job', async function () {
387 const startedAt = new Date()
389 const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID })
390 jobToken = job.jobToken
392 const checkProcessingJob = (job: RunnerJob & { jobToken?: string }, fromAccept: boolean) => {
393 expect(job.uuid).to.equal(jobUUID)
395 expect(job.type).to.equal('vod-web-video-transcoding')
396 expect(job.state.label).to.equal('Processing')
397 expect(job.state.id).to.equal(RunnerJobState.PROCESSING)
399 expect(job.runner).to.exist
400 expect(job.runner.name).to.equal('runner 1')
401 expect(job.runner.description).to.equal('my super runner 1')
403 expect(job.progress).to.be.null
405 expect(job.startedAt).to.exist
406 expect(new Date(job.startedAt)).to.be.above(startedAt)
408 expect(job.finishedAt).to.not.exist
410 expect(job.failures).to.equal(0)
412 expect(job.payload).to.exist
415 expect(job.jobToken).to.exist
416 expect((job as RunnerJobAdmin).privatePayload).to.not.exist
418 expect(job.jobToken).to.not.exist
419 expect((job as RunnerJobAdmin).privatePayload).to.exist
423 checkProcessingJob(job, true)
425 const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
427 const processingJob = data.find(j => j.uuid === jobUUID)
428 checkProcessingJob(processingJob, false)
430 await checkMainJobState(RunnerJobState.PROCESSING)
433 it('Should update a job', async function () {
434 await server.runnerJobs.update({ runnerToken, jobUUID, jobToken, progress: 53 })
436 const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
438 for (const job of data) {
439 if (job.state.id === RunnerJobState.PROCESSING) {
440 expect(job.progress).to.equal(53)
442 expect(job.progress).to.be.null
447 it('Should abort a job', async function () {
448 await server.runnerJobs.abort({ runnerToken, jobUUID, jobToken, reason: 'for tests' })
450 await checkMainJobState(RunnerJobState.PENDING)
452 const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
453 for (const job of data) {
454 expect(job.progress).to.be.null
458 it('Should accept the same job again and post a success', async function () {
459 const { availableJobs } = await server.runnerJobs.request({ runnerToken })
460 expect(availableJobs.find(j => j.uuid === jobUUID)).to.exist
462 const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID })
463 jobToken = job.jobToken
465 await checkMainJobState(RunnerJobState.PROCESSING)
467 const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
469 for (const job of data) {
470 expect(job.progress).to.be.null
474 videoFile: 'video_short.mp4'
477 await server.runnerJobs.success({ runnerToken, jobUUID, jobToken, payload })
480 it('Should not have available jobs anymore', async function () {
481 await checkMainJobState(RunnerJobState.COMPLETED)
483 const job = await getMainJob()
484 expect(job.finishedAt).to.exist
486 const { availableJobs } = await server.runnerJobs.request({ runnerToken })
487 expect(availableJobs.find(j => j.uuid === jobUUID)).to.not.exist
491 describe('Error job', function () {
493 it('Should accept another job and post an error', async function () {
494 await server.runnerJobs.cancelAllJobs()
495 await server.videos.quickUpload({ name: 'video' })
496 await waitJobs([ server ])
498 const { availableJobs } = await server.runnerJobs.request({ runnerToken })
499 jobUUID = availableJobs[0].uuid
501 const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID })
502 jobToken = job.jobToken
504 await server.runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' })
507 it('Should have job failures increased', async function () {
508 const job = await getMainJob()
509 expect(job.state.id).to.equal(RunnerJobState.PENDING)
510 expect(job.failures).to.equal(1)
511 expect(job.error).to.be.null
512 expect(job.progress).to.be.null
513 expect(job.finishedAt).to.not.exist
516 it('Should error a job when job attempts is too big', async function () {
517 for (let i = 0; i < 4; i++) {
518 const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID })
519 jobToken = job.jobToken
521 await server.runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error ' + i })
524 const job = await getMainJob()
525 expect(job.failures).to.equal(5)
526 expect(job.state.id).to.equal(RunnerJobState.ERRORED)
527 expect(job.state.label).to.equal('Errored')
528 expect(job.error).to.equal('Error 3')
529 expect(job.progress).to.be.null
530 expect(job.finishedAt).to.exist
535 it('Should have failed children jobs too', async function () {
536 const { data } = await server.runnerJobs.list({ count: 50, sort: '-updatedAt' })
538 const children = data.filter(j => j.parent?.uuid === failedJob.uuid)
539 expect(children).to.have.lengthOf(9)
541 for (const child of children) {
542 expect(child.parent.uuid).to.equal(failedJob.uuid)
543 expect(child.parent.type).to.equal(failedJob.type)
544 expect(child.parent.state.id).to.equal(failedJob.state.id)
545 expect(child.parent.state.label).to.equal(failedJob.state.label)
547 expect(child.state.id).to.equal(RunnerJobState.PARENT_ERRORED)
548 expect(child.state.label).to.equal('Parent job failed')
553 describe('Cancel', function () {
555 it('Should cancel a pending job', async function () {
556 await server.videos.quickUpload({ name: 'video' })
557 await waitJobs([ server ])
560 const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
562 const pendingJob = data.find(j => j.state.id === RunnerJobState.PENDING)
563 jobUUID = pendingJob.uuid
565 await server.runnerJobs.cancelByAdmin({ jobUUID })
569 const job = await getMainJob()
570 expect(job.state.id).to.equal(RunnerJobState.CANCELLED)
571 expect(job.state.label).to.equal('Cancelled')
575 const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
576 const children = data.filter(j => j.parent?.uuid === jobUUID)
577 expect(children).to.have.lengthOf(9)
579 for (const child of children) {
580 expect(child.state.id).to.equal(RunnerJobState.PARENT_CANCELLED)
585 it('Should cancel an already accepted job and skip success/error', async function () {
586 await server.videos.quickUpload({ name: 'video' })
587 await waitJobs([ server ])
589 const { availableJobs } = await server.runnerJobs.request({ runnerToken })
590 jobUUID = availableJobs[0].uuid
592 const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID })
593 jobToken = job.jobToken
595 await server.runnerJobs.cancelByAdmin({ jobUUID })
597 await server.runnerJobs.abort({ runnerToken, jobUUID, jobToken, reason: 'aborted', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
601 describe('Stalled jobs', function () {
603 it('Should abort stalled jobs', async function () {
606 await server.videos.quickUpload({ name: 'video' })
607 await server.videos.quickUpload({ name: 'video' })
608 await waitJobs([ server ])
610 const { job: job1 } = await server.runnerJobs.autoAccept({ runnerToken })
611 const { job: stalledJob } = await server.runnerJobs.autoAccept({ runnerToken })
613 for (let i = 0; i < 6; i++) {
616 await server.runnerJobs.update({ runnerToken, jobToken: job1.jobToken, jobUUID: job1.uuid })
619 const refreshedJob1 = await server.runnerJobs.getJob({ uuid: job1.uuid })
620 const refreshedStalledJob = await server.runnerJobs.getJob({ uuid: stalledJob.uuid })
622 expect(refreshedJob1.state.id).to.equal(RunnerJobState.PROCESSING)
623 expect(refreshedStalledJob.state.id).to.equal(RunnerJobState.PENDING)
627 describe('Rate limit', function () {
629 before(async function () {
643 it('Should rate limit an unknown runner, but not a registered one', async function () {
646 await server.videos.quickUpload({ name: 'video' })
647 await waitJobs([ server ])
649 const { job } = await server.runnerJobs.autoAccept({ runnerToken })
651 for (let i = 0; i < 20; i++) {
653 await server.runnerJobs.request({ runnerToken })
654 await server.runnerJobs.update({ runnerToken, jobToken: job.jobToken, jobUUID: job.uuid })
660 await server.runnerJobs.request({ runnerToken: 'toto', expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 })
661 await server.runnerJobs.update({
663 jobToken: job.jobToken,
665 expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429
671 await server.runnerJobs.request({ runnerToken: undefined, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 })
672 await server.runnerJobs.update({
673 runnerToken: undefined,
674 jobToken: job.jobToken,
676 expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429
682 await server.runnerJobs.request({ runnerToken })
683 await server.runnerJobs.update({ runnerToken, jobToken: job.jobToken, jobUUID: job.uuid })
689 after(async function () {
690 await cleanupTests([ server ])