From 5e47f6ab984a7d00782e4c7030afffa1ba480add Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 4 May 2023 15:29:34 +0200 Subject: Support studio transcoding in peertube runner --- server/tests/api/check-params/config.ts | 5 +- server/tests/api/check-params/runners.ts | 211 +++++++++++++++++++-- server/tests/api/runners/index.ts | 1 + server/tests/api/runners/runner-common.ts | 12 +- .../tests/api/runners/runner-studio-transcoding.ts | 168 ++++++++++++++++ server/tests/api/runners/runner-vod-transcoding.ts | 16 +- server/tests/api/server/config.ts | 7 +- server/tests/api/transcoding/video-studio.ts | 42 ++-- server/tests/peertube-runner/index.ts | 1 + server/tests/peertube-runner/live-transcoding.ts | 15 +- server/tests/peertube-runner/studio-transcoding.ts | 116 +++++++++++ server/tests/peertube-runner/vod-transcoding.ts | 14 +- server/tests/shared/checks.ts | 17 ++ server/tests/shared/directories.ts | 21 +- 14 files changed, 576 insertions(+), 70 deletions(-) create mode 100644 server/tests/api/runners/runner-studio-transcoding.ts create mode 100644 server/tests/peertube-runner/studio-transcoding.ts (limited to 'server/tests') diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index c5cda203e..472cad182 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -162,7 +162,10 @@ describe('Test config API validators', function () { } }, videoStudio: { - enabled: true + enabled: true, + remoteRunners: { + enabled: true + } }, import: { videos: { diff --git a/server/tests/api/check-params/runners.ts b/server/tests/api/check-params/runners.ts index 4da6fd91d..90a301392 100644 --- a/server/tests/api/check-params/runners.ts +++ b/server/tests/api/check-params/runners.ts @@ -1,6 +1,17 @@ +import { basename } from 'path' /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode, RunnerJob, RunnerJobState, RunnerJobSuccessPayload, RunnerJobUpdatePayload, VideoPrivacy } from '@shared/models' +import { + HttpStatusCode, + isVideoStudioTaskIntro, + RunnerJob, + RunnerJobState, + RunnerJobSuccessPayload, + RunnerJobUpdatePayload, + RunnerJobVideoEditionTranscodingPayload, + VideoPrivacy, + VideoStudioTaskIntro +} from '@shared/models' import { cleanupTests, createSingleServer, @@ -10,6 +21,7 @@ import { setAccessTokensToServers, setDefaultVideoChannel, stopFfmpeg, + VideoStudioCommand, waitJobs } from '@shared/server-commands' @@ -53,7 +65,10 @@ describe('Test managing runners', function () { registrationTokenId = data[0].id await server.config.enableTranscoding(true, true) + await server.config.enableStudio() await server.config.enableRemoteTranscoding() + await server.config.enableRemoteStudio() + runnerToken = await server.runners.autoRegisterRunner() runnerToken2 = await server.runners.autoRegisterRunner() @@ -249,6 +264,10 @@ describe('Test managing runners', function () { await server.runnerJobs.cancelByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) }) + it('Should fail with an already cancelled job', async function () { + await server.runnerJobs.cancelByAdmin({ jobUUID: cancelledJobUUID, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + it('Should succeed with the correct params', async function () { await server.runnerJobs.cancelByAdmin({ jobUUID }) }) @@ -296,9 +315,13 @@ describe('Test managing runners', function () { let pendingUUID: string + let videoStudioUUID: string + let studioFile: string + let liveAcceptedJob: RunnerJob & { jobToken: string } + let studioAcceptedJob: RunnerJob & { jobToken: string } - async function fetchFiles (options: { + async function fetchVideoInputFiles (options: { jobUUID: string videoUUID: string runnerToken: string @@ -315,6 +338,21 @@ describe('Test managing runners', function () { } } + async function fetchStudioFiles (options: { + jobUUID: string + videoUUID: string + runnerToken: string + jobToken: string + studioFile?: string + expectedStatus: HttpStatusCode + }) { + const { jobUUID, expectedStatus, videoUUID, runnerToken, jobToken, studioFile } = options + + const path = `/api/v1/runners/jobs/${jobUUID}/files/videos/${videoUUID}/studio/task-files/${studioFile}` + + await makePostBodyRequest({ url: server.url, path, fields: { runnerToken, jobToken }, expectedStatus }) + } + before(async function () { this.timeout(120000) @@ -352,6 +390,28 @@ describe('Test managing runners', function () { pendingUUID = availableJobs[0].uuid } + { + await server.config.disableTranscoding() + + const { uuid } = await server.videos.quickUpload({ name: 'video studio' }) + videoStudioUUID = uuid + + await server.config.enableTranscoding(true, true) + await server.config.enableStudio() + + await server.videoStudio.createEditionTasks({ + videoId: videoStudioUUID, + tasks: VideoStudioCommand.getComplexTask() + }) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'video-edition-transcoding' }) + studioAcceptedJob = job + + const tasks = (job.payload as RunnerJobVideoEditionTranscodingPayload).tasks + const fileUrl = (tasks.find(t => isVideoStudioTaskIntro(t)) as VideoStudioTaskIntro).options.file as string + studioFile = basename(fileUrl) + } + { await server.config.enableLive({ allowReplay: false, @@ -381,8 +441,6 @@ describe('Test managing runners', function () { jobToken: string expectedStatus: HttpStatusCode }) { - await fetchFiles({ ...options, videoUUID }) - await server.runnerJobs.abort({ ...options, reason: 'reason' }) await server.runnerJobs.update({ ...options }) await server.runnerJobs.error({ ...options, message: 'message' }) @@ -390,39 +448,95 @@ describe('Test managing runners', function () { } it('Should fail with an invalid job uuid', async function () { - await testEndpoints({ jobUUID: 'a', runnerToken, jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const options = { jobUUID: 'a', runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) + await fetchStudioFiles({ ...options, videoUUID, jobToken: studioAcceptedJob.jobToken, studioFile }) }) it('Should fail with an unknown job uuid', async function () { - const jobUUID = badUUID - await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const options = { jobUUID: badUUID, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) + await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID, studioFile }) }) it('Should fail with an invalid runner token', async function () { - await testEndpoints({ jobUUID, runnerToken: '', jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const options = { runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) }) it('Should fail with an unknown runner token', async function () { - const runnerToken = badUUID - await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const options = { runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) }) it('Should fail with an invalid job token job uuid', async function () { - await testEndpoints({ jobUUID, runnerToken, jobToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const options = { runnerToken, jobToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) }) it('Should fail with an unknown job token job uuid', async function () { - const jobToken = badUUID - await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const options = { runnerToken, jobToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) }) it('Should fail with a runner token not associated to this job', async function () { - await testEndpoints({ jobUUID, runnerToken: runnerToken2, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const options = { runnerToken: runnerToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) }) it('Should fail with a job uuid not associated to the job token', async function () { - await testEndpoints({ jobUUID: jobUUID2, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await testEndpoints({ jobUUID, runnerToken, jobToken: jobToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + { + const options = { jobUUID: jobUUID2, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, jobToken, videoUUID }) + await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID: videoStudioUUID, studioFile }) + } + + { + const options = { runnerToken, jobToken: jobToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) + } }) }) @@ -670,27 +784,82 @@ describe('Test managing runners', function () { }) }) }) + + describe('Video studio', function () { + + it('Should fail with an invalid video edition transcoding payload', async function () { + await server.runnerJobs.success({ + jobUUID: studioAcceptedJob.uuid, + jobToken: studioAcceptedJob.jobToken, + payload: { hello: 'video_short.mp4' } as any, + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) }) describe('Job files', function () { - describe('Video files', function () { + describe('Check video param for common job file routes', function () { + + async function fetchFiles (options: { + videoUUID?: string + expectedStatus: HttpStatusCode + }) { + await fetchVideoInputFiles({ videoUUID, ...options, jobToken, jobUUID, runnerToken }) + + await fetchStudioFiles({ + videoUUID: videoStudioUUID, + + ...options, + + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + runnerToken, + studioFile + }) + } it('Should fail with an invalid video id', async function () { - await fetchFiles({ videoUUID: 'a', jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await fetchFiles({ + videoUUID: 'a', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) }) it('Should fail with an unknown video id', async function () { const videoUUID = '910ec12a-d9e6-458b-a274-0abb655f9464' - await fetchFiles({ videoUUID, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + + await fetchFiles({ + videoUUID, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) }) it('Should fail with a video id not associated to this job', async function () { - await fetchFiles({ videoUUID: videoUUID2, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await fetchFiles({ + videoUUID: videoUUID2, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) }) it('Should succeed with the correct params', async function () { - await fetchFiles({ videoUUID, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.OK_200 }) + await fetchFiles({ expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('Video edition tasks file routes', function () { + + it('Should fail with an invalid studio filename', async function () { + await fetchStudioFiles({ + videoUUID: videoStudioUUID, + jobUUID: studioAcceptedJob.uuid, + runnerToken, + jobToken: studioAcceptedJob.jobToken, + studioFile: 'toto', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) }) }) }) diff --git a/server/tests/api/runners/index.ts b/server/tests/api/runners/index.ts index 7f33ec8dd..642a3a96d 100644 --- a/server/tests/api/runners/index.ts +++ b/server/tests/api/runners/index.ts @@ -1,4 +1,5 @@ export * from './runner-common' export * from './runner-live-transcoding' export * from './runner-socket' +export * from './runner-studio-transcoding' export * from './runner-vod-transcoding' diff --git a/server/tests/api/runners/runner-common.ts b/server/tests/api/runners/runner-common.ts index a2204753b..554024190 100644 --- a/server/tests/api/runners/runner-common.ts +++ b/server/tests/api/runners/runner-common.ts @@ -2,7 +2,15 @@ import { expect } from 'chai' import { wait } from '@shared/core-utils' -import { HttpStatusCode, Runner, RunnerJob, RunnerJobAdmin, RunnerJobState, RunnerRegistrationToken } from '@shared/models' +import { + HttpStatusCode, + Runner, + RunnerJob, + RunnerJobAdmin, + RunnerJobState, + RunnerJobVODWebVideoTranscodingPayload, + RunnerRegistrationToken +} from '@shared/models' import { cleanupTests, createSingleServer, @@ -349,7 +357,7 @@ describe('Test runner common actions', function () { for (const job of availableJobs) { expect(job.uuid).to.exist expect(job.payload.input).to.exist - expect(job.payload.output).to.exist + expect((job.payload as RunnerJobVODWebVideoTranscodingPayload).output).to.exist expect((job as RunnerJobAdmin).privatePayload).to.not.exist } diff --git a/server/tests/api/runners/runner-studio-transcoding.ts b/server/tests/api/runners/runner-studio-transcoding.ts new file mode 100644 index 000000000..9ae629be6 --- /dev/null +++ b/server/tests/api/runners/runner-studio-transcoding.ts @@ -0,0 +1,168 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { readFile } from 'fs-extra' +import { checkPersistentTmpIsEmpty, checkVideoDuration } from '@server/tests/shared' +import { buildAbsoluteFixturePath } from '@shared/core-utils' +import { + RunnerJobVideoEditionTranscodingPayload, + VideoEditionTranscodingSuccess, + VideoState, + VideoStudioTask, + VideoStudioTaskIntro +} from '@shared/models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + VideoStudioCommand, + waitJobs +} from '@shared/server-commands' + +describe('Test runner video studio transcoding', function () { + let servers: PeerTubeServer[] = [] + let runnerToken: string + let videoUUID: string + let jobUUID: string + + async function renewStudio (tasks: VideoStudioTask[] = VideoStudioCommand.getComplexTask()) { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = uuid + + await waitJobs(servers) + + await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks }) + await waitJobs(servers) + + const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) + expect(availableJobs).to.have.lengthOf(1) + + jobUUID = availableJobs[0].uuid + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableTranscoding(true, true) + await servers[0].config.enableStudio() + await servers[0].config.enableRemoteStudio() + + runnerToken = await servers[0].runners.autoRegisterRunner() + }) + + it('Should error a studio transcoding job', async function () { + this.timeout(60000) + + await renewStudio() + + for (let i = 0; i < 5; i++) { + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + await servers[0].runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) + } + + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.state.id).to.equal(VideoState.PUBLISHED) + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + it('Should cancel a transcoding job', async function () { + this.timeout(60000) + + await renewStudio() + + await servers[0].runnerJobs.cancelByAdmin({ jobUUID }) + + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.state.id).to.equal(VideoState.PUBLISHED) + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + it('Should execute a remote studio job', async function () { + this.timeout(240_000) + + const tasks = [ + { + name: 'add-outro' as 'add-outro', + options: { + file: 'video_short.webm' + } + }, + { + name: 'add-watermark' as 'add-watermark', + options: { + file: 'thumbnail.png' + } + }, + { + name: 'add-intro' as 'add-intro', + options: { + file: 'video_very_short_240p.mp4' + } + } + ] + + await renewStudio(tasks) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 5) + } + + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + expect(job.type === 'video-edition-transcoding') + expect(job.payload.input.videoFileUrl).to.exist + + // Check video input file + { + await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + } + + // Check task files + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i] + const payloadTask = job.payload.tasks[i] + + expect(payloadTask.name).to.equal(task.name) + + const inputFile = await readFile(buildAbsoluteFixturePath(task.options.file)) + + const { body } = await servers[0].runnerJobs.getJobFile({ + url: (payloadTask as VideoStudioTaskIntro).options.file as string, + jobToken, + runnerToken + }) + + expect(body).to.deep.equal(inputFile) + } + + const payload: VideoEditionTranscodingSuccess = { videoFile: 'video_very_short_240p.mp4' } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 2) + } + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/runners/runner-vod-transcoding.ts b/server/tests/api/runners/runner-vod-transcoding.ts index 92a47ac3b..b08ee312c 100644 --- a/server/tests/api/runners/runner-vod-transcoding.ts +++ b/server/tests/api/runners/runner-vod-transcoding.ts @@ -155,7 +155,7 @@ describe('Test runner VOD transcoding', function () { expect(job.payload.output.resolution).to.equal(720) expect(job.payload.output.fps).to.equal(25) - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short.webm')) expect(body).to.deep.equal(inputFile) @@ -200,7 +200,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) expect(body).to.deep.equal(inputFile) @@ -221,7 +221,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) expect(body).to.deep.equal(inputFile) @@ -293,7 +293,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) expect(body).to.deep.equal(inputFile) @@ -337,7 +337,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath(maxQualityFile)) expect(body).to.deep.equal(inputFile) @@ -446,13 +446,13 @@ describe('Test runner VOD transcoding', function () { expect(job.payload.output.resolution).to.equal(480) { - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.audioFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.audioFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('sample.ogg')) expect(body).to.deep.equal(inputFile) } { - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.previewFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.previewFileUrl, jobToken, runnerToken }) const video = await servers[0].videos.get({ id: videoUUID }) const { body: inputFile } = await makeGetRequest({ @@ -503,7 +503,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short_480p.mp4')) expect(body).to.deep.equal(inputFile) diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 54a40b994..011ba268c 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -102,6 +102,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.true expect(data.videoStudio.enabled).to.be.false + expect(data.videoStudio.remoteRunners.enabled).to.be.false expect(data.import.videos.concurrency).to.equal(2) expect(data.import.videos.http.enabled).to.be.true @@ -211,6 +212,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.false expect(data.videoStudio.enabled).to.be.true + expect(data.videoStudio.remoteRunners.enabled).to.be.true expect(data.import.videos.concurrency).to.equal(4) expect(data.import.videos.http.enabled).to.be.false @@ -374,7 +376,10 @@ const newCustomConfig: CustomConfig = { } }, videoStudio: { - enabled: true + enabled: true, + remoteRunners: { + enabled: true + } }, import: { videos: { diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts index 30f72e6e9..2f64ef6bd 100644 --- a/server/tests/api/transcoding/video-studio.ts +++ b/server/tests/api/transcoding/video-studio.ts @@ -1,5 +1,5 @@ import { expect } from 'chai' -import { checkPersistentTmpIsEmpty, expectStartWith } from '@server/tests/shared' +import { checkPersistentTmpIsEmpty, checkVideoDuration, expectStartWith } from '@server/tests/shared' import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' import { VideoStudioTask } from '@shared/models' import { @@ -18,20 +18,6 @@ describe('Test video studio', function () { let servers: PeerTubeServer[] = [] let videoUUID: string - async function checkDuration (server: PeerTubeServer, duration: number) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.duration).to.be.approximately(duration, 1) - - for (const file of video.files) { - const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) - - for (const stream of metadata.streams) { - expect(Math.round(stream.duration)).to.be.approximately(duration, 1) - } - } - } - async function renewVideo (fixture = 'video_short.webm') { const video = await servers[0].videos.quickUpload({ name: 'video', fixture }) videoUUID = video.uuid @@ -79,7 +65,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 3) + await checkVideoDuration(server, videoUUID, 3) const video = await server.videos.get({ id: videoUUID }) expect(new Date(video.publishedAt)).to.be.below(beforeTasks) @@ -100,7 +86,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 2) + await checkVideoDuration(server, videoUUID, 2) } }) @@ -119,7 +105,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 4) + await checkVideoDuration(server, videoUUID, 4) } }) }) @@ -140,7 +126,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 10) + await checkVideoDuration(server, videoUUID, 10) } }) @@ -158,7 +144,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 7) + await checkVideoDuration(server, videoUUID, 7) } }) @@ -183,7 +169,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 12) + await checkVideoDuration(server, videoUUID, 12) } }) @@ -201,7 +187,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 7) + await checkVideoDuration(server, videoUUID, 7) } }) @@ -219,7 +205,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 10) + await checkVideoDuration(server, videoUUID, 10) } }) @@ -237,7 +223,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 10) + await checkVideoDuration(server, videoUUID, 10) } }) }) @@ -279,7 +265,7 @@ describe('Test video studio', function () { await createTasks(VideoStudioCommand.getComplexTask()) for (const server of servers) { - await checkDuration(server, 9) + await checkVideoDuration(server, videoUUID, 9) } }) }) @@ -309,7 +295,7 @@ describe('Test video studio', function () { const video = await server.videos.get({ id: videoUUID }) expect(video.files).to.have.lengthOf(0) - await checkDuration(server, 9) + await checkVideoDuration(server, videoUUID, 9) } }) }) @@ -351,7 +337,7 @@ describe('Test video studio', function () { expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) } - await checkDuration(server, 9) + await checkVideoDuration(server, videoUUID, 9) } }) }) @@ -370,7 +356,7 @@ describe('Test video studio', function () { await waitJobs(servers) for (const server of servers) { - await checkDuration(server, 9) + await checkVideoDuration(server, videoUUID, 9) } }) diff --git a/server/tests/peertube-runner/index.ts b/server/tests/peertube-runner/index.ts index 6258d6eb2..470316417 100644 --- a/server/tests/peertube-runner/index.ts +++ b/server/tests/peertube-runner/index.ts @@ -1,3 +1,4 @@ export * from './client-cli' export * from './live-transcoding' +export * from './studio-transcoding' export * from './vod-transcoding' diff --git a/server/tests/peertube-runner/live-transcoding.ts b/server/tests/peertube-runner/live-transcoding.ts index f58e920ba..1e94eabcd 100644 --- a/server/tests/peertube-runner/live-transcoding.ts +++ b/server/tests/peertube-runner/live-transcoding.ts @@ -1,6 +1,12 @@ import { expect } from 'chai' /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expectStartWith, PeerTubeRunnerProcess, SQLCommand, testLiveVideoResolutions } from '@server/tests/shared' +import { + checkPeerTubeRunnerCacheIsEmpty, + expectStartWith, + PeerTubeRunnerProcess, + SQLCommand, + testLiveVideoResolutions +} from '@server/tests/shared' import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils' import { HttpStatusCode, VideoPrivacy } from '@shared/models' import { @@ -169,6 +175,13 @@ describe('Test Live transcoding in peertube-runner program', function () { runSuite({ objectStorage: true }) }) + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty() + }) + }) + after(async function () { await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) peertubeRunner.kill() diff --git a/server/tests/peertube-runner/studio-transcoding.ts b/server/tests/peertube-runner/studio-transcoding.ts new file mode 100644 index 000000000..cca905e2f --- /dev/null +++ b/server/tests/peertube-runner/studio-transcoding.ts @@ -0,0 +1,116 @@ + +import { expect } from 'chai' +import { checkPeerTubeRunnerCacheIsEmpty, checkVideoDuration, expectStartWith, PeerTubeRunnerProcess } from '@server/tests/shared' +import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + VideoStudioCommand, + waitJobs +} from '@shared/server-commands' + +describe('Test studio transcoding in peertube-runner program', function () { + let servers: PeerTubeServer[] = [] + let peertubeRunner: PeerTubeRunnerProcess + + function runSuite (options: { + objectStorage: boolean + }) { + const { objectStorage } = options + + it('Should run a complex studio transcoding', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) + + await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks: VideoStudioCommand.getComplexTask() }) + await waitJobs(servers, { runnerJobs: true }) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + const files = getAllFiles(video) + + for (const f of files) { + expect(oldFileUrls).to.not.include(f.fileUrl) + } + + if (objectStorage) { + for (const webtorrentFile of video.files) { + expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) + } + + for (const hlsFile of video.streamingPlaylists[0].files) { + expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) + } + } + + await checkVideoDuration(server, uuid, 9) + } + }) + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableTranscoding(true, true) + await servers[0].config.enableStudio() + await servers[0].config.enableRemoteStudio() + + const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() + + peertubeRunner = new PeerTubeRunnerProcess() + await peertubeRunner.runServer({ hideLogs: false }) + await peertubeRunner.registerPeerTubeInstance({ server: servers[0], registrationToken, runnerName: 'runner' }) + }) + + describe('With videos on local filesystem storage', function () { + runSuite({ objectStorage: false }) + }) + + describe('With videos on object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + before(async function () { + await ObjectStorageCommand.prepareDefaultMockBuckets() + + await servers[0].kill() + + await servers[0].run(ObjectStorageCommand.getDefaultMockConfig()) + + // Wait for peertube runner socket reconnection + await wait(1500) + }) + + runSuite({ objectStorage: true }) + }) + + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty() + }) + }) + + after(async function () { + await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) + peertubeRunner.kill() + + await cleanupTests(servers) + }) +}) diff --git a/server/tests/peertube-runner/vod-transcoding.ts b/server/tests/peertube-runner/vod-transcoding.ts index bdf798379..3a9abba93 100644 --- a/server/tests/peertube-runner/vod-transcoding.ts +++ b/server/tests/peertube-runner/vod-transcoding.ts @@ -1,6 +1,11 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { completeCheckHlsPlaylist, completeWebVideoFilesCheck, PeerTubeRunnerProcess } from '@server/tests/shared' +import { + checkPeerTubeRunnerCacheIsEmpty, + completeCheckHlsPlaylist, + completeWebVideoFilesCheck, + PeerTubeRunnerProcess +} from '@server/tests/shared' import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' import { VideoPrivacy } from '@shared/models' import { @@ -321,6 +326,13 @@ describe('Test VOD transcoding in peertube-runner program', function () { }) }) + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty() + }) + }) + after(async function () { await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) peertubeRunner.kill() diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index d7eb25bb5..feaef37c6 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -130,6 +130,22 @@ function checkBadSortPagination (url: string, path: string, token?: string, quer }) } +// --------------------------------------------------------------------------- + +async function checkVideoDuration (server: PeerTubeServer, videoUUID: string, duration: number) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.duration).to.be.approximately(duration, 1) + + for (const file of video.files) { + const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) + + for (const stream of metadata.streams) { + expect(Math.round(stream.duration)).to.be.approximately(duration, 1) + } + } +} + export { dateIsValid, testImageSize, @@ -142,5 +158,6 @@ export { checkBadStartPagination, checkBadCountPagination, checkBadSortPagination, + checkVideoDuration, expectLogContain } diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts index a614cef7c..4f4282554 100644 --- a/server/tests/shared/directories.ts +++ b/server/tests/shared/directories.ts @@ -2,9 +2,11 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' +import { homedir } from 'os' +import { join } from 'path' import { PeerTubeServer } from '@shared/server-commands' -async function checkTmpIsEmpty (server: PeerTubeServer) { +export async function checkTmpIsEmpty (server: PeerTubeServer) { await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) if (await pathExists(server.getDirectoryPath('tmp/hls'))) { @@ -12,11 +14,11 @@ async function checkTmpIsEmpty (server: PeerTubeServer) { } } -async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { +export async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { await checkDirectoryIsEmpty(server, 'tmp-persistent') } -async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { +export async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { const directoryPath = server.getDirectoryPath(directory) const directoryExists = await pathExists(directoryPath) @@ -28,8 +30,13 @@ async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, expect(filtered).to.have.lengthOf(0) } -export { - checkTmpIsEmpty, - checkPersistentTmpIsEmpty, - checkDirectoryIsEmpty +export async function checkPeerTubeRunnerCacheIsEmpty () { + const directoryPath = join(homedir(), '.cache', 'peertube-runner-nodejs', 'test', 'transcoding') + + const directoryExists = await pathExists(directoryPath) + expect(directoryExists).to.be.true + + const files = await readdir(directoryPath) + + expect(files).to.have.lengthOf(0) } -- cgit v1.2.3