diff options
67 files changed, 1425 insertions, 264 deletions
diff --git a/client/src/app/+admin/admin.component.ts b/client/src/app/+admin/admin.component.ts index d4d912c40..49092ea2a 100644 --- a/client/src/app/+admin/admin.component.ts +++ b/client/src/app/+admin/admin.component.ts | |||
@@ -272,6 +272,8 @@ export class AdminComponent implements OnInit { | |||
272 | private isRemoteRunnersEnabled () { | 272 | private isRemoteRunnersEnabled () { |
273 | const config = this.server.getHTMLConfig() | 273 | const config = this.server.getHTMLConfig() |
274 | 274 | ||
275 | return config.transcoding.remoteRunners.enabled || config.live.transcoding.remoteRunners.enabled | 275 | return config.transcoding.remoteRunners.enabled || |
276 | config.live.transcoding.remoteRunners.enabled || | ||
277 | config.videoStudio.remoteRunners.enabled | ||
276 | } | 278 | } |
277 | } | 279 | } |
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-configuration.service.ts b/client/src/app/+admin/config/edit-custom-config/edit-configuration.service.ts index 96f5b830e..6c431ce64 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-configuration.service.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-configuration.service.ts | |||
@@ -61,6 +61,10 @@ export class EditConfigurationService { | |||
61 | return form.value['transcoding']['enabled'] === true | 61 | return form.value['transcoding']['enabled'] === true |
62 | } | 62 | } |
63 | 63 | ||
64 | isStudioEnabled (form: FormGroup) { | ||
65 | return form.value['videoStudio']['enabled'] === true | ||
66 | } | ||
67 | |||
64 | isLiveEnabled (form: FormGroup) { | 68 | isLiveEnabled (form: FormGroup) { |
65 | return form.value['live']['enabled'] === true | 69 | return form.value['live']['enabled'] === true |
66 | } | 70 | } |
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index 335aedb67..30e4aa5d5 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts | |||
@@ -218,7 +218,10 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { | |||
218 | } | 218 | } |
219 | }, | 219 | }, |
220 | videoStudio: { | 220 | videoStudio: { |
221 | enabled: null | 221 | enabled: null, |
222 | remoteRunners: { | ||
223 | enabled: null | ||
224 | } | ||
222 | }, | 225 | }, |
223 | autoBlacklist: { | 226 | autoBlacklist: { |
224 | videos: { | 227 | videos: { |
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html index c11f560dd..b17c51532 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html | |||
@@ -230,6 +230,20 @@ | |||
230 | </ng-container> | 230 | </ng-container> |
231 | </my-peertube-checkbox> | 231 | </my-peertube-checkbox> |
232 | </div> | 232 | </div> |
233 | |||
234 | <div class="form-group" formGroupName="remoteRunners" [ngClass]="getStudioDisabledClass()"> | ||
235 | <my-peertube-checkbox | ||
236 | inputName="videoStudioRemoteRunnersEnabled" formControlName="enabled" | ||
237 | i18n-labelText labelText="Enable remote runners" | ||
238 | > | ||
239 | <ng-container ngProjectAs="description"> | ||
240 | <span i18n> | ||
241 | Use <a routerLink="/admin/system/runners/runners-list">remote runners</a> to process studio transcoding tasks. | ||
242 | Remote runners has to register on your instance first. | ||
243 | </span> | ||
244 | </ng-container> | ||
245 | </my-peertube-checkbox> | ||
246 | </div> | ||
233 | </ng-container> | 247 | </ng-container> |
234 | </div> | 248 | </div> |
235 | </div> | 249 | </div> |
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts index 184dfd921..e960533f9 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts | |||
@@ -62,10 +62,18 @@ export class EditVODTranscodingComponent implements OnInit, OnChanges { | |||
62 | return this.editConfigurationService.isTranscodingEnabled(this.form) | 62 | return this.editConfigurationService.isTranscodingEnabled(this.form) |
63 | } | 63 | } |
64 | 64 | ||
65 | isStudioEnabled () { | ||
66 | return this.editConfigurationService.isStudioEnabled(this.form) | ||
67 | } | ||
68 | |||
65 | getTranscodingDisabledClass () { | 69 | getTranscodingDisabledClass () { |
66 | return { 'disabled-checkbox-extra': !this.isTranscodingEnabled() } | 70 | return { 'disabled-checkbox-extra': !this.isTranscodingEnabled() } |
67 | } | 71 | } |
68 | 72 | ||
73 | getStudioDisabledClass () { | ||
74 | return { 'disabled-checkbox-extra': !this.isStudioEnabled() } | ||
75 | } | ||
76 | |||
69 | getTotalTranscodingThreads () { | 77 | getTotalTranscodingThreads () { |
70 | return this.editConfigurationService.getTotalTranscodingThreads(this.form) | 78 | return this.editConfigurationService.getTotalTranscodingThreads(this.form) |
71 | } | 79 | } |
diff --git a/config/default.yaml b/config/default.yaml index f3f29ecb9..14bb8d060 100644 --- a/config/default.yaml +++ b/config/default.yaml | |||
@@ -579,6 +579,12 @@ video_studio: | |||
579 | # If enabled, users can create transcoding tasks as they wish | 579 | # If enabled, users can create transcoding tasks as they wish |
580 | enabled: false | 580 | enabled: false |
581 | 581 | ||
582 | # Enable remote runners to transcode studio tasks | ||
583 | # If enabled, your instance won't transcode the videos itself | ||
584 | # At least 1 remote runner must be configured to transcode your videos | ||
585 | remote_runners: | ||
586 | enabled: false | ||
587 | |||
582 | import: | 588 | import: |
583 | # Add ability for your users to import remote videos (from YouTube, torrent...) | 589 | # Add ability for your users to import remote videos (from YouTube, torrent...) |
584 | videos: | 590 | videos: |
diff --git a/config/production.yaml.example b/config/production.yaml.example index ea6d77306..db9c18cb8 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example | |||
@@ -589,6 +589,13 @@ video_studio: | |||
589 | # If enabled, users can create transcoding tasks as they wish | 589 | # If enabled, users can create transcoding tasks as they wish |
590 | enabled: false | 590 | enabled: false |
591 | 591 | ||
592 | |||
593 | # Enable remote runners to transcode studio tasks | ||
594 | # If enabled, your instance won't transcode the videos itself | ||
595 | # At least 1 remote runner must be configured to transcode your videos | ||
596 | remote_runners: | ||
597 | enabled: false | ||
598 | |||
592 | import: | 599 | import: |
593 | # Add ability for your users to import remote videos (from YouTube, torrent...) | 600 | # Add ability for your users to import remote videos (from YouTube, torrent...) |
594 | videos: | 601 | videos: |
diff --git a/packages/peertube-runner/server/process/process.ts b/packages/peertube-runner/server/process/process.ts index 39a929c59..ef231cb38 100644 --- a/packages/peertube-runner/server/process/process.ts +++ b/packages/peertube-runner/server/process/process.ts | |||
@@ -1,12 +1,14 @@ | |||
1 | import { logger } from 'packages/peertube-runner/shared/logger' | 1 | import { logger } from 'packages/peertube-runner/shared/logger' |
2 | import { | 2 | import { |
3 | RunnerJobLiveRTMPHLSTranscodingPayload, | 3 | RunnerJobLiveRTMPHLSTranscodingPayload, |
4 | RunnerJobVideoEditionTranscodingPayload, | ||
4 | RunnerJobVODAudioMergeTranscodingPayload, | 5 | RunnerJobVODAudioMergeTranscodingPayload, |
5 | RunnerJobVODHLSTranscodingPayload, | 6 | RunnerJobVODHLSTranscodingPayload, |
6 | RunnerJobVODWebVideoTranscodingPayload | 7 | RunnerJobVODWebVideoTranscodingPayload |
7 | } from '@shared/models' | 8 | } from '@shared/models' |
8 | import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared' | 9 | import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared' |
9 | import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live' | 10 | import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live' |
11 | import { processStudioTranscoding } from './shared/process-studio' | ||
10 | 12 | ||
11 | export async function processJob (options: ProcessOptions) { | 13 | export async function processJob (options: ProcessOptions) { |
12 | const { server, job } = options | 14 | const { server, job } = options |
@@ -21,6 +23,8 @@ export async function processJob (options: ProcessOptions) { | |||
21 | await processHLSTranscoding(options as ProcessOptions<RunnerJobVODHLSTranscodingPayload>) | 23 | await processHLSTranscoding(options as ProcessOptions<RunnerJobVODHLSTranscodingPayload>) |
22 | } else if (job.type === 'live-rtmp-hls-transcoding') { | 24 | } else if (job.type === 'live-rtmp-hls-transcoding') { |
23 | await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>).process() | 25 | await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>).process() |
26 | } else if (job.type === 'video-edition-transcoding') { | ||
27 | await processStudioTranscoding(options as ProcessOptions<RunnerJobVideoEditionTranscodingPayload>) | ||
24 | } else { | 28 | } else { |
25 | logger.error(`Unknown job ${job.type} to process`) | 29 | logger.error(`Unknown job ${job.type} to process`) |
26 | return | 30 | return |
diff --git a/packages/peertube-runner/server/process/shared/common.ts b/packages/peertube-runner/server/process/shared/common.ts index 9b2c40728..3cac98388 100644 --- a/packages/peertube-runner/server/process/shared/common.ts +++ b/packages/peertube-runner/server/process/shared/common.ts | |||
@@ -2,11 +2,12 @@ import { throttle } from 'lodash' | |||
2 | import { ConfigManager, downloadFile, logger } from 'packages/peertube-runner/shared' | 2 | import { ConfigManager, downloadFile, logger } from 'packages/peertube-runner/shared' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { buildUUID } from '@shared/extra-utils' | 4 | import { buildUUID } from '@shared/extra-utils' |
5 | import { FFmpegLive, FFmpegVOD } from '@shared/ffmpeg' | 5 | import { FFmpegEdition, FFmpegLive, FFmpegVOD } from '@shared/ffmpeg' |
6 | import { RunnerJob, RunnerJobPayload } from '@shared/models' | 6 | import { RunnerJob, RunnerJobPayload } from '@shared/models' |
7 | import { PeerTubeServer } from '@shared/server-commands' | 7 | import { PeerTubeServer } from '@shared/server-commands' |
8 | import { getTranscodingLogger } from './transcoding-logger' | 8 | import { getTranscodingLogger } from './transcoding-logger' |
9 | import { getAvailableEncoders, getEncodersToTry } from './transcoding-profiles' | 9 | import { getAvailableEncoders, getEncodersToTry } from './transcoding-profiles' |
10 | import { remove } from 'fs-extra' | ||
10 | 11 | ||
11 | export type JobWithToken <T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string } | 12 | export type JobWithToken <T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string } |
12 | 13 | ||
@@ -24,7 +25,14 @@ export async function downloadInputFile (options: { | |||
24 | const { url, job, runnerToken } = options | 25 | const { url, job, runnerToken } = options |
25 | const destination = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID()) | 26 | const destination = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID()) |
26 | 27 | ||
27 | await downloadFile({ url, jobToken: job.jobToken, runnerToken, destination }) | 28 | try { |
29 | await downloadFile({ url, jobToken: job.jobToken, runnerToken, destination }) | ||
30 | } catch (err) { | ||
31 | remove(destination) | ||
32 | .catch(err => logger.error({ err }, `Cannot remove ${destination}`)) | ||
33 | |||
34 | throw err | ||
35 | } | ||
28 | 36 | ||
29 | return destination | 37 | return destination |
30 | } | 38 | } |
@@ -40,6 +48,8 @@ export async function updateTranscodingProgress (options: { | |||
40 | return server.runnerJobs.update({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken, progress }) | 48 | return server.runnerJobs.update({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken, progress }) |
41 | } | 49 | } |
42 | 50 | ||
51 | // --------------------------------------------------------------------------- | ||
52 | |||
43 | export function buildFFmpegVOD (options: { | 53 | export function buildFFmpegVOD (options: { |
44 | server: PeerTubeServer | 54 | server: PeerTubeServer |
45 | runnerToken: string | 55 | runnerToken: string |
@@ -58,26 +68,25 @@ export function buildFFmpegVOD (options: { | |||
58 | .catch(err => logger.error({ err }, 'Cannot send job progress')) | 68 | .catch(err => logger.error({ err }, 'Cannot send job progress')) |
59 | }, updateInterval, { trailing: false }) | 69 | }, updateInterval, { trailing: false }) |
60 | 70 | ||
61 | const config = ConfigManager.Instance.getConfig() | ||
62 | |||
63 | return new FFmpegVOD({ | 71 | return new FFmpegVOD({ |
64 | niceness: config.ffmpeg.nice, | 72 | ...getCommonFFmpegOptions(), |
65 | threads: config.ffmpeg.threads, | 73 | |
66 | tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), | ||
67 | profile: 'default', | ||
68 | availableEncoders: { | ||
69 | available: getAvailableEncoders(), | ||
70 | encodersToTry: getEncodersToTry() | ||
71 | }, | ||
72 | logger: getTranscodingLogger(), | ||
73 | updateJobProgress | 74 | updateJobProgress |
74 | }) | 75 | }) |
75 | } | 76 | } |
76 | 77 | ||
77 | export function buildFFmpegLive () { | 78 | export function buildFFmpegLive () { |
79 | return new FFmpegLive(getCommonFFmpegOptions()) | ||
80 | } | ||
81 | |||
82 | export function buildFFmpegEdition () { | ||
83 | return new FFmpegEdition(getCommonFFmpegOptions()) | ||
84 | } | ||
85 | |||
86 | function getCommonFFmpegOptions () { | ||
78 | const config = ConfigManager.Instance.getConfig() | 87 | const config = ConfigManager.Instance.getConfig() |
79 | 88 | ||
80 | return new FFmpegLive({ | 89 | return { |
81 | niceness: config.ffmpeg.nice, | 90 | niceness: config.ffmpeg.nice, |
82 | threads: config.ffmpeg.threads, | 91 | threads: config.ffmpeg.threads, |
83 | tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), | 92 | tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), |
@@ -87,5 +96,5 @@ export function buildFFmpegLive () { | |||
87 | encodersToTry: getEncodersToTry() | 96 | encodersToTry: getEncodersToTry() |
88 | }, | 97 | }, |
89 | logger: getTranscodingLogger() | 98 | logger: getTranscodingLogger() |
90 | }) | 99 | } |
91 | } | 100 | } |
diff --git a/packages/peertube-runner/server/process/shared/process-studio.ts b/packages/peertube-runner/server/process/shared/process-studio.ts new file mode 100644 index 000000000..f8262096e --- /dev/null +++ b/packages/peertube-runner/server/process/shared/process-studio.ts | |||
@@ -0,0 +1,138 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { pick } from 'lodash' | ||
3 | import { logger } from 'packages/peertube-runner/shared' | ||
4 | import { extname, join } from 'path' | ||
5 | import { buildUUID } from '@shared/extra-utils' | ||
6 | import { | ||
7 | RunnerJobVideoEditionTranscodingPayload, | ||
8 | VideoEditionTranscodingSuccess, | ||
9 | VideoStudioTask, | ||
10 | VideoStudioTaskCutPayload, | ||
11 | VideoStudioTaskIntroPayload, | ||
12 | VideoStudioTaskOutroPayload, | ||
13 | VideoStudioTaskPayload, | ||
14 | VideoStudioTaskWatermarkPayload | ||
15 | } from '@shared/models' | ||
16 | import { ConfigManager } from '../../../shared/config-manager' | ||
17 | import { buildFFmpegEdition, downloadInputFile, JobWithToken, ProcessOptions } from './common' | ||
18 | |||
19 | export async function processStudioTranscoding (options: ProcessOptions<RunnerJobVideoEditionTranscodingPayload>) { | ||
20 | const { server, job, runnerToken } = options | ||
21 | const payload = job.payload | ||
22 | |||
23 | let outputPath: string | ||
24 | const inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) | ||
25 | let tmpInputFilePath = inputPath | ||
26 | |||
27 | try { | ||
28 | for (const task of payload.tasks) { | ||
29 | const outputFilename = 'output-edition-' + buildUUID() + '.mp4' | ||
30 | outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), outputFilename) | ||
31 | |||
32 | await processTask({ | ||
33 | inputPath: tmpInputFilePath, | ||
34 | outputPath, | ||
35 | task, | ||
36 | job, | ||
37 | runnerToken | ||
38 | }) | ||
39 | |||
40 | if (tmpInputFilePath) await remove(tmpInputFilePath) | ||
41 | |||
42 | // For the next iteration | ||
43 | tmpInputFilePath = outputPath | ||
44 | } | ||
45 | |||
46 | const successBody: VideoEditionTranscodingSuccess = { | ||
47 | videoFile: outputPath | ||
48 | } | ||
49 | |||
50 | await server.runnerJobs.success({ | ||
51 | jobToken: job.jobToken, | ||
52 | jobUUID: job.uuid, | ||
53 | runnerToken, | ||
54 | payload: successBody | ||
55 | }) | ||
56 | } finally { | ||
57 | await remove(tmpInputFilePath) | ||
58 | await remove(outputPath) | ||
59 | } | ||
60 | } | ||
61 | |||
62 | // --------------------------------------------------------------------------- | ||
63 | // Private | ||
64 | // --------------------------------------------------------------------------- | ||
65 | |||
66 | type TaskProcessorOptions <T extends VideoStudioTaskPayload = VideoStudioTaskPayload> = { | ||
67 | inputPath: string | ||
68 | outputPath: string | ||
69 | task: T | ||
70 | runnerToken: string | ||
71 | job: JobWithToken | ||
72 | } | ||
73 | |||
74 | const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise<any> } = { | ||
75 | 'add-intro': processAddIntroOutro, | ||
76 | 'add-outro': processAddIntroOutro, | ||
77 | 'cut': processCut, | ||
78 | 'add-watermark': processAddWatermark | ||
79 | } | ||
80 | |||
81 | async function processTask (options: TaskProcessorOptions) { | ||
82 | const { task } = options | ||
83 | |||
84 | const processor = taskProcessors[options.task.name] | ||
85 | if (!process) throw new Error('Unknown task ' + task.name) | ||
86 | |||
87 | return processor(options) | ||
88 | } | ||
89 | |||
90 | async function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) { | ||
91 | const { inputPath, task, runnerToken, job } = options | ||
92 | |||
93 | logger.debug('Adding intro/outro to ' + inputPath) | ||
94 | |||
95 | const introOutroPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) | ||
96 | |||
97 | return buildFFmpegEdition().addIntroOutro({ | ||
98 | ...pick(options, [ 'inputPath', 'outputPath' ]), | ||
99 | |||
100 | introOutroPath, | ||
101 | type: task.name === 'add-intro' | ||
102 | ? 'intro' | ||
103 | : 'outro' | ||
104 | }) | ||
105 | } | ||
106 | |||
107 | function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) { | ||
108 | const { inputPath, task } = options | ||
109 | |||
110 | logger.debug(`Cutting ${inputPath}`) | ||
111 | |||
112 | return buildFFmpegEdition().cutVideo({ | ||
113 | ...pick(options, [ 'inputPath', 'outputPath' ]), | ||
114 | |||
115 | start: task.options.start, | ||
116 | end: task.options.end | ||
117 | }) | ||
118 | } | ||
119 | |||
120 | async function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) { | ||
121 | const { inputPath, task, runnerToken, job } = options | ||
122 | |||
123 | logger.debug('Adding watermark to ' + inputPath) | ||
124 | |||
125 | const watermarkPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) | ||
126 | |||
127 | return buildFFmpegEdition().addWatermark({ | ||
128 | ...pick(options, [ 'inputPath', 'outputPath' ]), | ||
129 | |||
130 | watermarkPath, | ||
131 | |||
132 | videoFilters: { | ||
133 | watermarkSizeRatio: task.options.watermarkSizeRatio, | ||
134 | horitonzalMarginRatio: task.options.horitonzalMarginRatio, | ||
135 | verticalMarginRatio: task.options.verticalMarginRatio | ||
136 | } | ||
137 | }) | ||
138 | } | ||
diff --git a/packages/peertube-runner/server/process/shared/process-vod.ts b/packages/peertube-runner/server/process/shared/process-vod.ts index aae61e9c5..d84ece3cb 100644 --- a/packages/peertube-runner/server/process/shared/process-vod.ts +++ b/packages/peertube-runner/server/process/shared/process-vod.ts | |||
@@ -62,33 +62,36 @@ export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVO | |||
62 | 62 | ||
63 | const ffmpegVod = buildFFmpegVOD({ job, server, runnerToken }) | 63 | const ffmpegVod = buildFFmpegVOD({ job, server, runnerToken }) |
64 | 64 | ||
65 | await ffmpegVod.transcode({ | 65 | try { |
66 | type: 'hls', | 66 | await ffmpegVod.transcode({ |
67 | copyCodecs: false, | 67 | type: 'hls', |
68 | inputPath, | 68 | copyCodecs: false, |
69 | hlsPlaylist: { videoFilename }, | 69 | inputPath, |
70 | outputPath, | 70 | hlsPlaylist: { videoFilename }, |
71 | 71 | outputPath, | |
72 | inputFileMutexReleaser: () => {}, | 72 | |
73 | 73 | inputFileMutexReleaser: () => {}, | |
74 | resolution: payload.output.resolution, | 74 | |
75 | fps: payload.output.fps | 75 | resolution: payload.output.resolution, |
76 | }) | 76 | fps: payload.output.fps |
77 | 77 | }) | |
78 | const successBody: VODHLSTranscodingSuccess = { | 78 | |
79 | resolutionPlaylistFile: outputPath, | 79 | const successBody: VODHLSTranscodingSuccess = { |
80 | videoFile: videoPath | 80 | resolutionPlaylistFile: outputPath, |
81 | videoFile: videoPath | ||
82 | } | ||
83 | |||
84 | await server.runnerJobs.success({ | ||
85 | jobToken: job.jobToken, | ||
86 | jobUUID: job.uuid, | ||
87 | runnerToken, | ||
88 | payload: successBody | ||
89 | }) | ||
90 | } finally { | ||
91 | await remove(inputPath) | ||
92 | await remove(outputPath) | ||
93 | await remove(videoPath) | ||
81 | } | 94 | } |
82 | |||
83 | await server.runnerJobs.success({ | ||
84 | jobToken: job.jobToken, | ||
85 | jobUUID: job.uuid, | ||
86 | runnerToken, | ||
87 | payload: successBody | ||
88 | }) | ||
89 | |||
90 | await remove(outputPath) | ||
91 | await remove(videoPath) | ||
92 | } | 95 | } |
93 | 96 | ||
94 | export async function processAudioMergeTranscoding (options: ProcessOptions<RunnerJobVODAudioMergeTranscodingPayload>) { | 97 | export async function processAudioMergeTranscoding (options: ProcessOptions<RunnerJobVODAudioMergeTranscodingPayload>) { |
diff --git a/packages/peertube-runner/server/server.ts b/packages/peertube-runner/server/server.ts index e851dfc7c..8eff4bd2f 100644 --- a/packages/peertube-runner/server/server.ts +++ b/packages/peertube-runner/server/server.ts | |||
@@ -8,6 +8,7 @@ import { ConfigManager } from '../shared' | |||
8 | import { IPCServer } from '../shared/ipc' | 8 | import { IPCServer } from '../shared/ipc' |
9 | import { logger } from '../shared/logger' | 9 | import { logger } from '../shared/logger' |
10 | import { JobWithToken, processJob } from './process' | 10 | import { JobWithToken, processJob } from './process' |
11 | import { isJobSupported } from './shared' | ||
11 | 12 | ||
12 | type PeerTubeServer = PeerTubeServerCommand & { | 13 | type PeerTubeServer = PeerTubeServerCommand & { |
13 | runnerToken: string | 14 | runnerToken: string |
@@ -199,12 +200,14 @@ export class RunnerServer { | |||
199 | 200 | ||
200 | const { availableJobs } = await server.runnerJobs.request({ runnerToken: server.runnerToken }) | 201 | const { availableJobs } = await server.runnerJobs.request({ runnerToken: server.runnerToken }) |
201 | 202 | ||
202 | if (availableJobs.length === 0) { | 203 | const filtered = availableJobs.filter(j => isJobSupported(j)) |
204 | |||
205 | if (filtered.length === 0) { | ||
203 | logger.debug(`No job available on ${server.url} for runner ${server.runnerName}`) | 206 | logger.debug(`No job available on ${server.url} for runner ${server.runnerName}`) |
204 | return undefined | 207 | return undefined |
205 | } | 208 | } |
206 | 209 | ||
207 | return availableJobs[0] | 210 | return filtered[0] |
208 | } | 211 | } |
209 | 212 | ||
210 | private async tryToExecuteJobAsync (server: PeerTubeServer, jobToAccept: { uuid: string }) { | 213 | private async tryToExecuteJobAsync (server: PeerTubeServer, jobToAccept: { uuid: string }) { |
diff --git a/packages/peertube-runner/server/shared/index.ts b/packages/peertube-runner/server/shared/index.ts new file mode 100644 index 000000000..5c86bafc0 --- /dev/null +++ b/packages/peertube-runner/server/shared/index.ts | |||
@@ -0,0 +1 @@ | |||
export * from './supported-job' | |||
diff --git a/packages/peertube-runner/server/shared/supported-job.ts b/packages/peertube-runner/server/shared/supported-job.ts new file mode 100644 index 000000000..87d5a39cc --- /dev/null +++ b/packages/peertube-runner/server/shared/supported-job.ts | |||
@@ -0,0 +1,43 @@ | |||
1 | import { | ||
2 | RunnerJobLiveRTMPHLSTranscodingPayload, | ||
3 | RunnerJobPayload, | ||
4 | RunnerJobType, | ||
5 | RunnerJobVideoEditionTranscodingPayload, | ||
6 | RunnerJobVODAudioMergeTranscodingPayload, | ||
7 | RunnerJobVODHLSTranscodingPayload, | ||
8 | RunnerJobVODWebVideoTranscodingPayload, | ||
9 | VideoStudioTaskPayload | ||
10 | } from '@shared/models' | ||
11 | |||
12 | const supportedMatrix = { | ||
13 | 'vod-web-video-transcoding': (_payload: RunnerJobVODWebVideoTranscodingPayload) => { | ||
14 | return true | ||
15 | }, | ||
16 | 'vod-hls-transcoding': (_payload: RunnerJobVODHLSTranscodingPayload) => { | ||
17 | return true | ||
18 | }, | ||
19 | 'vod-audio-merge-transcoding': (_payload: RunnerJobVODAudioMergeTranscodingPayload) => { | ||
20 | return true | ||
21 | }, | ||
22 | 'live-rtmp-hls-transcoding': (_payload: RunnerJobLiveRTMPHLSTranscodingPayload) => { | ||
23 | return true | ||
24 | }, | ||
25 | 'video-edition-transcoding': (payload: RunnerJobVideoEditionTranscodingPayload) => { | ||
26 | const tasks = payload?.tasks | ||
27 | const supported = new Set<VideoStudioTaskPayload['name']>([ 'add-intro', 'add-outro', 'add-watermark', 'cut' ]) | ||
28 | |||
29 | if (!Array.isArray(tasks)) return false | ||
30 | |||
31 | return tasks.every(t => t && supported.has(t.name)) | ||
32 | } | ||
33 | } | ||
34 | |||
35 | export function isJobSupported (job: { | ||
36 | type: RunnerJobType | ||
37 | payload: RunnerJobPayload | ||
38 | }) { | ||
39 | const fn = supportedMatrix[job.type] | ||
40 | if (!fn) return false | ||
41 | |||
42 | return fn(job.payload as any) | ||
43 | } | ||
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 0b9aaffda..3b6230f4a 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts | |||
@@ -274,7 +274,10 @@ function customConfig (): CustomConfig { | |||
274 | } | 274 | } |
275 | }, | 275 | }, |
276 | videoStudio: { | 276 | videoStudio: { |
277 | enabled: CONFIG.VIDEO_STUDIO.ENABLED | 277 | enabled: CONFIG.VIDEO_STUDIO.ENABLED, |
278 | remoteRunners: { | ||
279 | enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED | ||
280 | } | ||
278 | }, | 281 | }, |
279 | import: { | 282 | import: { |
280 | videos: { | 283 | videos: { |
diff --git a/server/controllers/api/runners/jobs-files.ts b/server/controllers/api/runners/jobs-files.ts index e43ce35f5..4efa40b3a 100644 --- a/server/controllers/api/runners/jobs-files.ts +++ b/server/controllers/api/runners/jobs-files.ts | |||
@@ -2,9 +2,13 @@ import express from 'express' | |||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
3 | import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage' | 3 | import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage' |
4 | import { VideoPathManager } from '@server/lib/video-path-manager' | 4 | import { VideoPathManager } from '@server/lib/video-path-manager' |
5 | import { getStudioTaskFilePath } from '@server/lib/video-studio' | ||
5 | import { asyncMiddleware } from '@server/middlewares' | 6 | import { asyncMiddleware } from '@server/middlewares' |
6 | import { jobOfRunnerGetValidator } from '@server/middlewares/validators/runners' | 7 | import { jobOfRunnerGetValidator } from '@server/middlewares/validators/runners' |
7 | import { runnerJobGetVideoTranscodingFileValidator } from '@server/middlewares/validators/runners/job-files' | 8 | import { |
9 | runnerJobGetVideoStudioTaskFileValidator, | ||
10 | runnerJobGetVideoTranscodingFileValidator | ||
11 | } from '@server/middlewares/validators/runners/job-files' | ||
8 | import { VideoStorage } from '@shared/models' | 12 | import { VideoStorage } from '@shared/models' |
9 | 13 | ||
10 | const lTags = loggerTagsFactory('api', 'runner') | 14 | const lTags = loggerTagsFactory('api', 'runner') |
@@ -23,6 +27,13 @@ runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/previews/max-qua | |||
23 | getMaxQualityVideoPreview | 27 | getMaxQualityVideoPreview |
24 | ) | 28 | ) |
25 | 29 | ||
30 | runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/studio/task-files/:filename', | ||
31 | asyncMiddleware(jobOfRunnerGetValidator), | ||
32 | asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), | ||
33 | runnerJobGetVideoStudioTaskFileValidator, | ||
34 | getVideoEditionTaskFile | ||
35 | ) | ||
36 | |||
26 | // --------------------------------------------------------------------------- | 37 | // --------------------------------------------------------------------------- |
27 | 38 | ||
28 | export { | 39 | export { |
@@ -82,3 +93,17 @@ function getMaxQualityVideoPreview (req: express.Request, res: express.Response) | |||
82 | 93 | ||
83 | return res.sendFile(file.getPath()) | 94 | return res.sendFile(file.getPath()) |
84 | } | 95 | } |
96 | |||
97 | function getVideoEditionTaskFile (req: express.Request, res: express.Response) { | ||
98 | const runnerJob = res.locals.runnerJob | ||
99 | const runner = runnerJob.Runner | ||
100 | const video = res.locals.videoAll | ||
101 | const filename = req.params.filename | ||
102 | |||
103 | logger.info( | ||
104 | 'Get video edition task file %s of video %s of job %s for runner %s', filename, video.uuid, runnerJob.uuid, runner.name, | ||
105 | lTags(runner.name, runnerJob.id, runnerJob.type) | ||
106 | ) | ||
107 | |||
108 | return res.sendFile(getStudioTaskFilePath(filename)) | ||
109 | } | ||
diff --git a/server/controllers/api/runners/jobs.ts b/server/controllers/api/runners/jobs.ts index 7d488ec11..8e34c07a3 100644 --- a/server/controllers/api/runners/jobs.ts +++ b/server/controllers/api/runners/jobs.ts | |||
@@ -17,6 +17,7 @@ import { | |||
17 | import { | 17 | import { |
18 | abortRunnerJobValidator, | 18 | abortRunnerJobValidator, |
19 | acceptRunnerJobValidator, | 19 | acceptRunnerJobValidator, |
20 | cancelRunnerJobValidator, | ||
20 | errorRunnerJobValidator, | 21 | errorRunnerJobValidator, |
21 | getRunnerFromTokenValidator, | 22 | getRunnerFromTokenValidator, |
22 | jobOfRunnerGetValidator, | 23 | jobOfRunnerGetValidator, |
@@ -41,6 +42,7 @@ import { | |||
41 | RunnerJobUpdateBody, | 42 | RunnerJobUpdateBody, |
42 | RunnerJobUpdatePayload, | 43 | RunnerJobUpdatePayload, |
43 | UserRight, | 44 | UserRight, |
45 | VideoEditionTranscodingSuccess, | ||
44 | VODAudioMergeTranscodingSuccess, | 46 | VODAudioMergeTranscodingSuccess, |
45 | VODHLSTranscodingSuccess, | 47 | VODHLSTranscodingSuccess, |
46 | VODWebVideoTranscodingSuccess | 48 | VODWebVideoTranscodingSuccess |
@@ -110,6 +112,7 @@ runnerJobsRouter.post('/jobs/:jobUUID/cancel', | |||
110 | authenticate, | 112 | authenticate, |
111 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), | 113 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), |
112 | asyncMiddleware(runnerJobGetValidator), | 114 | asyncMiddleware(runnerJobGetValidator), |
115 | cancelRunnerJobValidator, | ||
113 | asyncMiddleware(cancelRunnerJob) | 116 | asyncMiddleware(cancelRunnerJob) |
114 | ) | 117 | ) |
115 | 118 | ||
@@ -297,6 +300,14 @@ const jobSuccessPayloadBuilders: { | |||
297 | } | 300 | } |
298 | }, | 301 | }, |
299 | 302 | ||
303 | 'video-edition-transcoding': (payload: VideoEditionTranscodingSuccess, files) => { | ||
304 | return { | ||
305 | ...payload, | ||
306 | |||
307 | videoFile: files['payload[videoFile]'][0].path | ||
308 | } | ||
309 | }, | ||
310 | |||
300 | 'live-rtmp-hls-transcoding': () => ({}) | 311 | 'live-rtmp-hls-transcoding': () => ({}) |
301 | } | 312 | } |
302 | 313 | ||
@@ -327,7 +338,7 @@ async function postRunnerJobSuccess (req: express.Request, res: express.Response | |||
327 | async function cancelRunnerJob (req: express.Request, res: express.Response) { | 338 | async function cancelRunnerJob (req: express.Request, res: express.Response) { |
328 | const runnerJob = res.locals.runnerJob | 339 | const runnerJob = res.locals.runnerJob |
329 | 340 | ||
330 | logger.info('Cancelling job %s (%s)', runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) | 341 | logger.info('Cancelling job %s (%s)', runnerJob.uuid, runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) |
331 | 342 | ||
332 | const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) | 343 | const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) |
333 | await new RunnerJobHandler().cancel({ runnerJob }) | 344 | await new RunnerJobHandler().cancel({ runnerJob }) |
diff --git a/server/controllers/api/videos/studio.ts b/server/controllers/api/videos/studio.ts index 2ccb2fb89..7c31dfd2b 100644 --- a/server/controllers/api/videos/studio.ts +++ b/server/controllers/api/videos/studio.ts | |||
@@ -1,12 +1,10 @@ | |||
1 | import Bluebird from 'bluebird' | 1 | import Bluebird from 'bluebird' |
2 | import express from 'express' | 2 | import express from 'express' |
3 | import { move } from 'fs-extra' | 3 | import { move } from 'fs-extra' |
4 | import { basename, join } from 'path' | 4 | import { basename } from 'path' |
5 | import { createAnyReqFiles } from '@server/helpers/express-utils' | 5 | import { createAnyReqFiles } from '@server/helpers/express-utils' |
6 | import { CONFIG } from '@server/initializers/config' | 6 | import { MIMETYPES, VIDEO_FILTERS } from '@server/initializers/constants' |
7 | import { MIMETYPES } from '@server/initializers/constants' | 7 | import { buildTaskFileFieldname, createVideoStudioJob, getStudioTaskFilePath, getTaskFileFromReq } from '@server/lib/video-studio' |
8 | import { JobQueue } from '@server/lib/job-queue' | ||
9 | import { buildTaskFileFieldname, getTaskFileFromReq } from '@server/lib/video-studio' | ||
10 | import { | 8 | import { |
11 | HttpStatusCode, | 9 | HttpStatusCode, |
12 | VideoState, | 10 | VideoState, |
@@ -75,7 +73,11 @@ async function createEditionTasks (req: express.Request, res: express.Response) | |||
75 | tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files)) | 73 | tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files)) |
76 | } | 74 | } |
77 | 75 | ||
78 | JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload }) | 76 | await createVideoStudioJob({ |
77 | user: res.locals.oauth.token.User, | ||
78 | payload, | ||
79 | video | ||
80 | }) | ||
79 | 81 | ||
80 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | 82 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) |
81 | } | 83 | } |
@@ -124,13 +126,16 @@ async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: numbe | |||
124 | return { | 126 | return { |
125 | name: task.name, | 127 | name: task.name, |
126 | options: { | 128 | options: { |
127 | file: destination | 129 | file: destination, |
130 | watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO, | ||
131 | horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO, | ||
132 | verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO | ||
128 | } | 133 | } |
129 | } | 134 | } |
130 | } | 135 | } |
131 | 136 | ||
132 | async function moveStudioFileToPersistentTMP (file: string) { | 137 | async function moveStudioFileToPersistentTMP (file: string) { |
133 | const destination = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, basename(file)) | 138 | const destination = getStudioTaskFilePath(basename(file)) |
134 | 139 | ||
135 | await move(file, destination) | 140 | await move(file, destination) |
136 | 141 | ||
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index fa0f469f6..2c4cd1b9f 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts | |||
@@ -15,8 +15,12 @@ function isSafePath (p: string) { | |||
15 | }) | 15 | }) |
16 | } | 16 | } |
17 | 17 | ||
18 | function isSafeFilename (filename: string, extension: string) { | 18 | function isSafeFilename (filename: string, extension?: string) { |
19 | return typeof filename === 'string' && !!filename.match(new RegExp(`^[a-z0-9-]+\\.${extension}$`)) | 19 | const regex = extension |
20 | ? new RegExp(`^[a-z0-9-]+\\.${extension}$`) | ||
21 | : new RegExp(`^[a-z0-9-]+\\.[a-z0-9]{1,8}$`) | ||
22 | |||
23 | return typeof filename === 'string' && !!filename.match(regex) | ||
20 | } | 24 | } |
21 | 25 | ||
22 | function isSafePeerTubeFilenameWithoutExtension (filename: string) { | 26 | function isSafePeerTubeFilenameWithoutExtension (filename: string) { |
diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts index 5f755d5bb..934bd37c9 100644 --- a/server/helpers/custom-validators/runners/jobs.ts +++ b/server/helpers/custom-validators/runners/jobs.ts | |||
@@ -6,6 +6,7 @@ import { | |||
6 | RunnerJobSuccessPayload, | 6 | RunnerJobSuccessPayload, |
7 | RunnerJobType, | 7 | RunnerJobType, |
8 | RunnerJobUpdatePayload, | 8 | RunnerJobUpdatePayload, |
9 | VideoEditionTranscodingSuccess, | ||
9 | VODAudioMergeTranscodingSuccess, | 10 | VODAudioMergeTranscodingSuccess, |
10 | VODHLSTranscodingSuccess, | 11 | VODHLSTranscodingSuccess, |
11 | VODWebVideoTranscodingSuccess | 12 | VODWebVideoTranscodingSuccess |
@@ -23,7 +24,8 @@ function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: R | |||
23 | return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) || | 24 | return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) || |
24 | isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || | 25 | isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || |
25 | isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || | 26 | isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || |
26 | isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) | 27 | isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) || |
28 | isRunnerJobVideoEditionResultPayloadValid(value as VideoEditionTranscodingSuccess, type, files) | ||
27 | } | 29 | } |
28 | 30 | ||
29 | // --------------------------------------------------------------------------- | 31 | // --------------------------------------------------------------------------- |
@@ -35,6 +37,7 @@ function isRunnerJobProgressValid (value: string) { | |||
35 | function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) { | 37 | function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) { |
36 | return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) || | 38 | return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) || |
37 | isRunnerJobVODHLSUpdatePayloadValid(value, type, files) || | 39 | isRunnerJobVODHLSUpdatePayloadValid(value, type, files) || |
40 | isRunnerJobVideoEditionUpdatePayloadValid(value, type, files) || | ||
38 | isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) || | 41 | isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) || |
39 | isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) | 42 | isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) |
40 | } | 43 | } |
@@ -102,6 +105,15 @@ function isRunnerJobLiveRTMPHLSResultPayloadValid ( | |||
102 | return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | 105 | return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0)) |
103 | } | 106 | } |
104 | 107 | ||
108 | function isRunnerJobVideoEditionResultPayloadValid ( | ||
109 | _value: VideoEditionTranscodingSuccess, | ||
110 | type: RunnerJobType, | ||
111 | files: UploadFilesForCheck | ||
112 | ) { | ||
113 | return type === 'video-edition-transcoding' && | ||
114 | isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) | ||
115 | } | ||
116 | |||
105 | // --------------------------------------------------------------------------- | 117 | // --------------------------------------------------------------------------- |
106 | 118 | ||
107 | function isRunnerJobVODWebVideoUpdatePayloadValid ( | 119 | function isRunnerJobVODWebVideoUpdatePayloadValid ( |
@@ -164,3 +176,12 @@ function isRunnerJobLiveRTMPHLSUpdatePayloadValid ( | |||
164 | ) | 176 | ) |
165 | ) | 177 | ) |
166 | } | 178 | } |
179 | |||
180 | function isRunnerJobVideoEditionUpdatePayloadValid ( | ||
181 | value: RunnerJobUpdatePayload, | ||
182 | type: RunnerJobType, | ||
183 | _files: UploadFilesForCheck | ||
184 | ) { | ||
185 | return type === 'video-edition-transcoding' && | ||
186 | (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
187 | } | ||
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 2361aa1eb..2f5a274e4 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -38,7 +38,7 @@ function checkMissedConfig () { | |||
38 | 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', | 38 | 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', |
39 | 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', | 39 | 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', |
40 | 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', | 40 | 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', |
41 | 'video_studio.enabled', | 41 | 'video_studio.enabled', 'video_studio.remote_runners.enabled', |
42 | 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', | 42 | 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', |
43 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', | 43 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', |
44 | 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', | 44 | 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index f2d8f99b5..9c2705689 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -423,7 +423,10 @@ const CONFIG = { | |||
423 | } | 423 | } |
424 | }, | 424 | }, |
425 | VIDEO_STUDIO: { | 425 | VIDEO_STUDIO: { |
426 | get ENABLED () { return config.get<boolean>('video_studio.enabled') } | 426 | get ENABLED () { return config.get<boolean>('video_studio.enabled') }, |
427 | REMOTE_RUNNERS: { | ||
428 | get ENABLED () { return config.get<boolean>('video_studio.remote_runners.enabled') } | ||
429 | } | ||
427 | }, | 430 | }, |
428 | IMPORT: { | 431 | IMPORT: { |
429 | VIDEOS: { | 432 | VIDEOS: { |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 279e77421..6a757a0ff 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -229,7 +229,8 @@ const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = { | |||
229 | } | 229 | } |
230 | } | 230 | } |
231 | const JOB_PRIORITY = { | 231 | const JOB_PRIORITY = { |
232 | TRANSCODING: 100 | 232 | TRANSCODING: 100, |
233 | VIDEO_STUDIO: 150 | ||
233 | } | 234 | } |
234 | 235 | ||
235 | const JOB_REMOVAL_OPTIONS = { | 236 | const JOB_REMOVAL_OPTIONS = { |
diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts index 5e8dd4f51..df73caf72 100644 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/lib/job-queue/handlers/video-studio-edition.ts | |||
@@ -1,25 +1,18 @@ | |||
1 | import { Job } from 'bullmq' | 1 | import { Job } from 'bullmq' |
2 | import { move, remove } from 'fs-extra' | 2 | import { remove } from 'fs-extra' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' | 4 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' |
5 | import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' | ||
6 | import { CONFIG } from '@server/initializers/config' | 5 | import { CONFIG } from '@server/initializers/config' |
7 | import { VIDEO_FILTERS } from '@server/initializers/constants' | ||
8 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
9 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | ||
10 | import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' | ||
11 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' | 6 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' |
12 | import { isAbleToUploadVideo } from '@server/lib/user' | 7 | import { isAbleToUploadVideo } from '@server/lib/user' |
13 | import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' | ||
14 | import { VideoPathManager } from '@server/lib/video-path-manager' | 8 | import { VideoPathManager } from '@server/lib/video-path-manager' |
15 | import { approximateIntroOutroAdditionalSize, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' | 9 | import { approximateIntroOutroAdditionalSize, onVideoEditionEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' |
16 | import { UserModel } from '@server/models/user/user' | 10 | import { UserModel } from '@server/models/user/user' |
17 | import { VideoModel } from '@server/models/video/video' | 11 | import { VideoModel } from '@server/models/video/video' |
18 | import { VideoFileModel } from '@server/models/video/video-file' | 12 | import { MVideo, MVideoFullLight } from '@server/types/models' |
19 | import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models' | 13 | import { pick } from '@shared/core-utils' |
20 | import { getLowercaseExtension, pick } from '@shared/core-utils' | 14 | import { buildUUID } from '@shared/extra-utils' |
21 | import { buildUUID, getFileSize } from '@shared/extra-utils' | 15 | import { FFmpegEdition } from '@shared/ffmpeg' |
22 | import { FFmpegEdition, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' | ||
23 | import { | 16 | import { |
24 | VideoStudioEditionPayload, | 17 | VideoStudioEditionPayload, |
25 | VideoStudioTask, | 18 | VideoStudioTask, |
@@ -46,7 +39,7 @@ async function processVideoStudioEdition (job: Job) { | |||
46 | if (!video) { | 39 | if (!video) { |
47 | logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) | 40 | logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) |
48 | 41 | ||
49 | await safeCleanupStudioTMPFiles(payload) | 42 | await safeCleanupStudioTMPFiles(payload.tasks) |
50 | return undefined | 43 | return undefined |
51 | } | 44 | } |
52 | 45 | ||
@@ -81,28 +74,9 @@ async function processVideoStudioEdition (job: Job) { | |||
81 | 74 | ||
82 | logger.info('Video edition ended for video %s.', video.uuid, lTags) | 75 | logger.info('Video edition ended for video %s.', video.uuid, lTags) |
83 | 76 | ||
84 | const newFile = await buildNewFile(video, editionResultPath) | 77 | await onVideoEditionEnded({ video, editionResultPath, tasks: payload.tasks }) |
85 | |||
86 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) | ||
87 | await move(editionResultPath, outputPath) | ||
88 | |||
89 | await safeCleanupStudioTMPFiles(payload) | ||
90 | |||
91 | await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) | ||
92 | await removeAllFiles(video, newFile) | ||
93 | |||
94 | await newFile.save() | ||
95 | |||
96 | video.duration = await getVideoStreamDuration(outputPath) | ||
97 | await video.save() | ||
98 | |||
99 | await federateVideoIfNeeded(video, false, undefined) | ||
100 | |||
101 | const user = await UserModel.loadByVideoId(video.id) | ||
102 | |||
103 | await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) | ||
104 | } catch (err) { | 78 | } catch (err) { |
105 | await safeCleanupStudioTMPFiles(payload) | 79 | await safeCleanupStudioTMPFiles(payload.tasks) |
106 | 80 | ||
107 | throw err | 81 | throw err |
108 | } | 82 | } |
@@ -181,44 +155,15 @@ function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWater | |||
181 | watermarkPath: task.options.file, | 155 | watermarkPath: task.options.file, |
182 | 156 | ||
183 | videoFilters: { | 157 | videoFilters: { |
184 | watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO, | 158 | watermarkSizeRatio: task.options.watermarkSizeRatio, |
185 | horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO, | 159 | horitonzalMarginRatio: task.options.horitonzalMarginRatio, |
186 | verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO | 160 | verticalMarginRatio: task.options.verticalMarginRatio |
187 | } | 161 | } |
188 | }) | 162 | }) |
189 | } | 163 | } |
190 | 164 | ||
191 | // --------------------------------------------------------------------------- | 165 | // --------------------------------------------------------------------------- |
192 | 166 | ||
193 | async function buildNewFile (video: MVideoId, path: string) { | ||
194 | const videoFile = new VideoFileModel({ | ||
195 | extname: getLowercaseExtension(path), | ||
196 | size: await getFileSize(path), | ||
197 | metadata: await buildFileMetadata(path), | ||
198 | videoStreamingPlaylistId: null, | ||
199 | videoId: video.id | ||
200 | }) | ||
201 | |||
202 | const probe = await ffprobePromise(path) | ||
203 | |||
204 | videoFile.fps = await getVideoStreamFPS(path, probe) | ||
205 | videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution | ||
206 | |||
207 | videoFile.filename = generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname) | ||
208 | |||
209 | return videoFile | ||
210 | } | ||
211 | |||
212 | async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) { | ||
213 | await removeHLSPlaylist(video) | ||
214 | |||
215 | for (const file of video.VideoFiles) { | ||
216 | if (file.id === webTorrentFileException.id) continue | ||
217 | |||
218 | await removeWebTorrentFile(video, file.id) | ||
219 | } | ||
220 | } | ||
221 | |||
222 | async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) { | 167 | async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) { |
223 | const user = await UserModel.loadByVideoId(video.id) | 168 | const user = await UserModel.loadByVideoId(video.id) |
224 | 169 | ||
diff --git a/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/lib/runners/job-handlers/abstract-job-handler.ts index 74b455107..76fd1c5ac 100644 --- a/server/lib/runners/job-handlers/abstract-job-handler.ts +++ b/server/lib/runners/job-handlers/abstract-job-handler.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import { throttle } from 'lodash' | ||
1 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | 2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' |
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
3 | import { RUNNER_JOBS } from '@server/initializers/constants' | 4 | import { RUNNER_JOBS } from '@server/initializers/constants' |
@@ -14,6 +15,8 @@ import { | |||
14 | RunnerJobSuccessPayload, | 15 | RunnerJobSuccessPayload, |
15 | RunnerJobType, | 16 | RunnerJobType, |
16 | RunnerJobUpdatePayload, | 17 | RunnerJobUpdatePayload, |
18 | RunnerJobVideoEditionTranscodingPayload, | ||
19 | RunnerJobVideoEditionTranscodingPrivatePayload, | ||
17 | RunnerJobVODAudioMergeTranscodingPayload, | 20 | RunnerJobVODAudioMergeTranscodingPayload, |
18 | RunnerJobVODAudioMergeTranscodingPrivatePayload, | 21 | RunnerJobVODAudioMergeTranscodingPrivatePayload, |
19 | RunnerJobVODHLSTranscodingPayload, | 22 | RunnerJobVODHLSTranscodingPayload, |
@@ -21,7 +24,6 @@ import { | |||
21 | RunnerJobVODWebVideoTranscodingPayload, | 24 | RunnerJobVODWebVideoTranscodingPayload, |
22 | RunnerJobVODWebVideoTranscodingPrivatePayload | 25 | RunnerJobVODWebVideoTranscodingPrivatePayload |
23 | } from '@shared/models' | 26 | } from '@shared/models' |
24 | import { throttle } from 'lodash' | ||
25 | 27 | ||
26 | type CreateRunnerJobArg = | 28 | type CreateRunnerJobArg = |
27 | { | 29 | { |
@@ -43,6 +45,11 @@ type CreateRunnerJobArg = | |||
43 | type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'> | 45 | type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'> |
44 | payload: RunnerJobLiveRTMPHLSTranscodingPayload | 46 | payload: RunnerJobLiveRTMPHLSTranscodingPayload |
45 | privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload | 47 | privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload |
48 | } | | ||
49 | { | ||
50 | type: Extract<RunnerJobType, 'video-edition-transcoding'> | ||
51 | payload: RunnerJobVideoEditionTranscodingPayload | ||
52 | privatePayload: RunnerJobVideoEditionTranscodingPrivatePayload | ||
46 | } | 53 | } |
47 | 54 | ||
48 | export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> { | 55 | export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> { |
@@ -62,6 +69,8 @@ export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S | |||
62 | }): Promise<MRunnerJob> { | 69 | }): Promise<MRunnerJob> { |
63 | const { priority, dependsOnRunnerJob } = options | 70 | const { priority, dependsOnRunnerJob } = options |
64 | 71 | ||
72 | logger.debug('Creating runner job', { options, ...this.lTags(options.type) }) | ||
73 | |||
65 | const runnerJob = new RunnerJobModel({ | 74 | const runnerJob = new RunnerJobModel({ |
66 | ...pick(options, [ 'type', 'payload', 'privatePayload' ]), | 75 | ...pick(options, [ 'type', 'payload', 'privatePayload' ]), |
67 | 76 | ||
diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts index 517645848..a910ae383 100644 --- a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts | |||
@@ -4,27 +4,19 @@ import { logger } from '@server/helpers/logger' | |||
4 | import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' | 4 | import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' |
5 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | 5 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
6 | import { MRunnerJob } from '@server/types/models/runners' | 6 | import { MRunnerJob } from '@server/types/models/runners' |
7 | import { | 7 | import { RunnerJobSuccessPayload, RunnerJobUpdatePayload, RunnerJobVODPrivatePayload } from '@shared/models' |
8 | LiveRTMPHLSTranscodingUpdatePayload, | ||
9 | RunnerJobSuccessPayload, | ||
10 | RunnerJobUpdatePayload, | ||
11 | RunnerJobVODPrivatePayload | ||
12 | } from '@shared/models' | ||
13 | import { AbstractJobHandler } from './abstract-job-handler' | 8 | import { AbstractJobHandler } from './abstract-job-handler' |
14 | import { loadTranscodingRunnerVideo } from './shared' | 9 | import { loadTranscodingRunnerVideo } from './shared' |
15 | 10 | ||
16 | // eslint-disable-next-line max-len | 11 | // eslint-disable-next-line max-len |
17 | export abstract class AbstractVODTranscodingJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> extends AbstractJobHandler<C, U, S> { | 12 | export abstract class AbstractVODTranscodingJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> extends AbstractJobHandler<C, U, S> { |
18 | 13 | ||
19 | // --------------------------------------------------------------------------- | ||
20 | |||
21 | protected isAbortSupported () { | 14 | protected isAbortSupported () { |
22 | return true | 15 | return true |
23 | } | 16 | } |
24 | 17 | ||
25 | protected specificUpdate (_options: { | 18 | protected specificUpdate (_options: { |
26 | runnerJob: MRunnerJob | 19 | runnerJob: MRunnerJob |
27 | updatePayload?: LiveRTMPHLSTranscodingUpdatePayload | ||
28 | }) { | 20 | }) { |
29 | // empty | 21 | // empty |
30 | } | 22 | } |
diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts index 0fca72b9a..a40cee865 100644 --- a/server/lib/runners/job-handlers/index.ts +++ b/server/lib/runners/job-handlers/index.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | export * from './abstract-job-handler' | 1 | export * from './abstract-job-handler' |
2 | export * from './live-rtmp-hls-transcoding-job-handler' | 2 | export * from './live-rtmp-hls-transcoding-job-handler' |
3 | export * from './runner-job-handlers' | ||
4 | export * from './video-edition-transcoding-job-handler' | ||
3 | export * from './vod-audio-merge-transcoding-job-handler' | 5 | export * from './vod-audio-merge-transcoding-job-handler' |
4 | export * from './vod-hls-transcoding-job-handler' | 6 | export * from './vod-hls-transcoding-job-handler' |
5 | export * from './vod-web-video-transcoding-job-handler' | 7 | export * from './vod-web-video-transcoding-job-handler' |
6 | export * from './runner-job-handlers' | ||
diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts index c3d0e427d..48a70d891 100644 --- a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts | |||
@@ -70,7 +70,7 @@ export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateO | |||
70 | 70 | ||
71 | // --------------------------------------------------------------------------- | 71 | // --------------------------------------------------------------------------- |
72 | 72 | ||
73 | async specificUpdate (options: { | 73 | protected async specificUpdate (options: { |
74 | runnerJob: MRunnerJob | 74 | runnerJob: MRunnerJob |
75 | updatePayload: LiveRTMPHLSTranscodingUpdatePayload | 75 | updatePayload: LiveRTMPHLSTranscodingUpdatePayload |
76 | }) { | 76 | }) { |
diff --git a/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/lib/runners/job-handlers/runner-job-handlers.ts index 7bad1bc77..4ea6684ea 100644 --- a/server/lib/runners/job-handlers/runner-job-handlers.ts +++ b/server/lib/runners/job-handlers/runner-job-handlers.ts | |||
@@ -2,6 +2,7 @@ import { MRunnerJob } from '@server/types/models/runners' | |||
2 | import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models' | 2 | import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models' |
3 | import { AbstractJobHandler } from './abstract-job-handler' | 3 | import { AbstractJobHandler } from './abstract-job-handler' |
4 | import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler' | 4 | import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler' |
5 | import { VideoEditionTranscodingJobHandler } from './video-edition-transcoding-job-handler' | ||
5 | import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler' | 6 | import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler' |
6 | import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler' | 7 | import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler' |
7 | import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler' | 8 | import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler' |
@@ -10,7 +11,8 @@ const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, Run | |||
10 | 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, | 11 | 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, |
11 | 'vod-hls-transcoding': VODHLSTranscodingJobHandler, | 12 | 'vod-hls-transcoding': VODHLSTranscodingJobHandler, |
12 | 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, | 13 | 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, |
13 | 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler | 14 | 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler, |
15 | 'video-edition-transcoding': VideoEditionTranscodingJobHandler | ||
14 | } | 16 | } |
15 | 17 | ||
16 | export function getRunnerJobHandlerClass (job: MRunnerJob) { | 18 | export function getRunnerJobHandlerClass (job: MRunnerJob) { |
diff --git a/server/lib/runners/job-handlers/video-edition-transcoding-job-handler.ts b/server/lib/runners/job-handlers/video-edition-transcoding-job-handler.ts new file mode 100644 index 000000000..39a755c48 --- /dev/null +++ b/server/lib/runners/job-handlers/video-edition-transcoding-job-handler.ts | |||
@@ -0,0 +1,157 @@ | |||
1 | |||
2 | import { basename } from 'path' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { onVideoEditionEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' | ||
5 | import { MVideo } from '@server/types/models' | ||
6 | import { MRunnerJob } from '@server/types/models/runners' | ||
7 | import { buildUUID } from '@shared/extra-utils' | ||
8 | import { | ||
9 | isVideoStudioTaskIntro, | ||
10 | isVideoStudioTaskOutro, | ||
11 | isVideoStudioTaskWatermark, | ||
12 | RunnerJobState, | ||
13 | RunnerJobUpdatePayload, | ||
14 | RunnerJobVideoEditionTranscodingPayload, | ||
15 | RunnerJobVideoEditionTranscodingPrivatePayload, | ||
16 | VideoEditionTranscodingSuccess, | ||
17 | VideoState, | ||
18 | VideoStudioTaskPayload | ||
19 | } from '@shared/models' | ||
20 | import { generateRunnerEditionTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' | ||
21 | import { AbstractJobHandler } from './abstract-job-handler' | ||
22 | import { loadTranscodingRunnerVideo } from './shared' | ||
23 | |||
24 | type CreateOptions = { | ||
25 | video: MVideo | ||
26 | tasks: VideoStudioTaskPayload[] | ||
27 | priority: number | ||
28 | } | ||
29 | |||
30 | // eslint-disable-next-line max-len | ||
31 | export class VideoEditionTranscodingJobHandler extends AbstractJobHandler<CreateOptions, RunnerJobUpdatePayload, VideoEditionTranscodingSuccess> { | ||
32 | |||
33 | async create (options: CreateOptions) { | ||
34 | const { video, priority, tasks } = options | ||
35 | |||
36 | const jobUUID = buildUUID() | ||
37 | const payload: RunnerJobVideoEditionTranscodingPayload = { | ||
38 | input: { | ||
39 | videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) | ||
40 | }, | ||
41 | tasks: tasks.map(t => { | ||
42 | if (isVideoStudioTaskIntro(t) || isVideoStudioTaskOutro(t)) { | ||
43 | return { | ||
44 | ...t, | ||
45 | |||
46 | options: { | ||
47 | ...t.options, | ||
48 | |||
49 | file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) | ||
50 | } | ||
51 | } | ||
52 | } | ||
53 | |||
54 | if (isVideoStudioTaskWatermark(t)) { | ||
55 | return { | ||
56 | ...t, | ||
57 | |||
58 | options: { | ||
59 | ...t.options, | ||
60 | |||
61 | file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) | ||
62 | } | ||
63 | } | ||
64 | } | ||
65 | |||
66 | return t | ||
67 | }) | ||
68 | } | ||
69 | |||
70 | const privatePayload: RunnerJobVideoEditionTranscodingPrivatePayload = { | ||
71 | videoUUID: video.uuid, | ||
72 | originalTasks: tasks | ||
73 | } | ||
74 | |||
75 | const job = await this.createRunnerJob({ | ||
76 | type: 'video-edition-transcoding', | ||
77 | jobUUID, | ||
78 | payload, | ||
79 | privatePayload, | ||
80 | priority | ||
81 | }) | ||
82 | |||
83 | return job | ||
84 | } | ||
85 | |||
86 | // --------------------------------------------------------------------------- | ||
87 | |||
88 | protected isAbortSupported () { | ||
89 | return true | ||
90 | } | ||
91 | |||
92 | protected specificUpdate (_options: { | ||
93 | runnerJob: MRunnerJob | ||
94 | }) { | ||
95 | // empty | ||
96 | } | ||
97 | |||
98 | protected specificAbort (_options: { | ||
99 | runnerJob: MRunnerJob | ||
100 | }) { | ||
101 | // empty | ||
102 | } | ||
103 | |||
104 | protected async specificComplete (options: { | ||
105 | runnerJob: MRunnerJob | ||
106 | resultPayload: VideoEditionTranscodingSuccess | ||
107 | }) { | ||
108 | const { runnerJob, resultPayload } = options | ||
109 | const privatePayload = runnerJob.privatePayload as RunnerJobVideoEditionTranscodingPrivatePayload | ||
110 | |||
111 | const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) | ||
112 | if (!video) { | ||
113 | await safeCleanupStudioTMPFiles(privatePayload.originalTasks) | ||
114 | |||
115 | } | ||
116 | |||
117 | const videoFilePath = resultPayload.videoFile as string | ||
118 | |||
119 | await onVideoEditionEnded({ video, editionResultPath: videoFilePath, tasks: privatePayload.originalTasks }) | ||
120 | |||
121 | logger.info( | ||
122 | 'Runner video edition transcoding job %s for %s ended.', | ||
123 | runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) | ||
124 | ) | ||
125 | } | ||
126 | |||
127 | protected specificError (options: { | ||
128 | runnerJob: MRunnerJob | ||
129 | nextState: RunnerJobState | ||
130 | }) { | ||
131 | if (options.nextState === RunnerJobState.ERRORED) { | ||
132 | return this.specificErrorOrCancel(options) | ||
133 | } | ||
134 | |||
135 | return Promise.resolve() | ||
136 | } | ||
137 | |||
138 | protected specificCancel (options: { | ||
139 | runnerJob: MRunnerJob | ||
140 | }) { | ||
141 | return this.specificErrorOrCancel(options) | ||
142 | } | ||
143 | |||
144 | private async specificErrorOrCancel (options: { | ||
145 | runnerJob: MRunnerJob | ||
146 | }) { | ||
147 | const { runnerJob } = options | ||
148 | |||
149 | const payload = runnerJob.privatePayload as RunnerJobVideoEditionTranscodingPrivatePayload | ||
150 | await safeCleanupStudioTMPFiles(payload.originalTasks) | ||
151 | |||
152 | const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) | ||
153 | if (!video) return | ||
154 | |||
155 | return video.setNewState(VideoState.PUBLISHED, false, undefined) | ||
156 | } | ||
157 | } | ||
diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts index a7b33f87e..5f247d792 100644 --- a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts | |||
@@ -64,7 +64,7 @@ export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJo | |||
64 | 64 | ||
65 | // --------------------------------------------------------------------------- | 65 | // --------------------------------------------------------------------------- |
66 | 66 | ||
67 | async specificComplete (options: { | 67 | protected async specificComplete (options: { |
68 | runnerJob: MRunnerJob | 68 | runnerJob: MRunnerJob |
69 | resultPayload: VODAudioMergeTranscodingSuccess | 69 | resultPayload: VODAudioMergeTranscodingSuccess |
70 | }) { | 70 | }) { |
diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts index 02566b9d5..cc94bcbda 100644 --- a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts | |||
@@ -71,7 +71,7 @@ export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandle | |||
71 | 71 | ||
72 | // --------------------------------------------------------------------------- | 72 | // --------------------------------------------------------------------------- |
73 | 73 | ||
74 | async specificComplete (options: { | 74 | protected async specificComplete (options: { |
75 | runnerJob: MRunnerJob | 75 | runnerJob: MRunnerJob |
76 | resultPayload: VODHLSTranscodingSuccess | 76 | resultPayload: VODHLSTranscodingSuccess |
77 | }) { | 77 | }) { |
diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts index 57761a7a1..663d3306e 100644 --- a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts | |||
@@ -62,7 +62,7 @@ export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobH | |||
62 | 62 | ||
63 | // --------------------------------------------------------------------------- | 63 | // --------------------------------------------------------------------------- |
64 | 64 | ||
65 | async specificComplete (options: { | 65 | protected async specificComplete (options: { |
66 | runnerJob: MRunnerJob | 66 | runnerJob: MRunnerJob |
67 | resultPayload: VODWebVideoTranscodingSuccess | 67 | resultPayload: VODWebVideoTranscodingSuccess |
68 | }) { | 68 | }) { |
diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts index 329fb1170..a27060b33 100644 --- a/server/lib/runners/runner-urls.ts +++ b/server/lib/runners/runner-urls.ts | |||
@@ -7,3 +7,7 @@ export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, vid | |||
7 | export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { | 7 | export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { |
8 | return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' | 8 | return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' |
9 | } | 9 | } |
10 | |||
11 | export function generateRunnerEditionTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string, filename: string) { | ||
12 | return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/studio/task-files/' + filename | ||
13 | } | ||
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index ba7916363..924adb337 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts | |||
@@ -166,7 +166,10 @@ class ServerConfigManager { | |||
166 | } | 166 | } |
167 | }, | 167 | }, |
168 | videoStudio: { | 168 | videoStudio: { |
169 | enabled: CONFIG.VIDEO_STUDIO.ENABLED | 169 | enabled: CONFIG.VIDEO_STUDIO.ENABLED, |
170 | remoteRunners: { | ||
171 | enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED | ||
172 | } | ||
170 | }, | 173 | }, |
171 | import: { | 174 | import: { |
172 | videos: { | 175 | videos: { |
diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts index 576e786d5..80dc05bfb 100644 --- a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts +++ b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts | |||
@@ -1,6 +1,4 @@ | |||
1 | 1 | ||
2 | import { JOB_PRIORITY } from '@server/initializers/constants' | ||
3 | import { VideoModel } from '@server/models/video/video' | ||
4 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' | 2 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' |
5 | 3 | ||
6 | export abstract class AbstractJobBuilder { | 4 | export abstract class AbstractJobBuilder { |
@@ -20,20 +18,4 @@ export abstract class AbstractJobBuilder { | |||
20 | isNewVideo: boolean | 18 | isNewVideo: boolean |
21 | user: MUserId | null | 19 | user: MUserId | null |
22 | }): Promise<any> | 20 | }): Promise<any> |
23 | |||
24 | protected async getTranscodingJobPriority (options: { | ||
25 | user: MUserId | ||
26 | fallback: number | ||
27 | }) { | ||
28 | const { user, fallback } = options | ||
29 | |||
30 | if (!user) return fallback | ||
31 | |||
32 | const now = new Date() | ||
33 | const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) | ||
34 | |||
35 | const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) | ||
36 | |||
37 | return JOB_PRIORITY.TRANSCODING + videoUploadedByUser | ||
38 | } | ||
39 | } | 21 | } |
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts index 5a9c93ee5..29ee2ca61 100644 --- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts +++ b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts | |||
@@ -16,6 +16,7 @@ import { | |||
16 | OptimizeTranscodingPayload, | 16 | OptimizeTranscodingPayload, |
17 | VideoTranscodingPayload | 17 | VideoTranscodingPayload |
18 | } from '@shared/models' | 18 | } from '@shared/models' |
19 | import { getTranscodingJobPriority } from '../../transcoding-priority' | ||
19 | import { canDoQuickTranscode } from '../../transcoding-quick-transcode' | 20 | import { canDoQuickTranscode } from '../../transcoding-quick-transcode' |
20 | import { computeResolutionsToTranscode } from '../../transcoding-resolutions' | 21 | import { computeResolutionsToTranscode } from '../../transcoding-resolutions' |
21 | import { AbstractJobBuilder } from './abstract-job-builder' | 22 | import { AbstractJobBuilder } from './abstract-job-builder' |
@@ -178,7 +179,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
178 | 179 | ||
179 | return { | 180 | return { |
180 | type: 'video-transcoding' as 'video-transcoding', | 181 | type: 'video-transcoding' as 'video-transcoding', |
181 | priority: await this.getTranscodingJobPriority({ user, fallback: undefined }), | 182 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }), |
182 | payload | 183 | payload |
183 | } | 184 | } |
184 | } | 185 | } |
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts index 274dce21b..90b035402 100644 --- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts | |||
@@ -8,6 +8,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager' | |||
8 | import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' | 8 | import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' |
9 | import { MRunnerJob } from '@server/types/models/runners' | 9 | import { MRunnerJob } from '@server/types/models/runners' |
10 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' | 10 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' |
11 | import { getTranscodingJobPriority } from '../../transcoding-priority' | ||
11 | import { computeResolutionsToTranscode } from '../../transcoding-resolutions' | 12 | import { computeResolutionsToTranscode } from '../../transcoding-resolutions' |
12 | import { AbstractJobBuilder } from './abstract-job-builder' | 13 | import { AbstractJobBuilder } from './abstract-job-builder' |
13 | 14 | ||
@@ -49,7 +50,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
49 | : resolution | 50 | : resolution |
50 | 51 | ||
51 | const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) | 52 | const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) |
52 | const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) | 53 | const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) |
53 | 54 | ||
54 | const mainRunnerJob = videoFile.isAudio() | 55 | const mainRunnerJob = videoFile.isAudio() |
55 | ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) | 56 | ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) |
@@ -63,7 +64,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
63 | fps, | 64 | fps, |
64 | isNewVideo, | 65 | isNewVideo, |
65 | dependsOnRunnerJob: mainRunnerJob, | 66 | dependsOnRunnerJob: mainRunnerJob, |
66 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | 67 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) |
67 | }) | 68 | }) |
68 | } | 69 | } |
69 | 70 | ||
@@ -96,7 +97,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
96 | const maxResolution = Math.max(...resolutions) | 97 | const maxResolution = Math.max(...resolutions) |
97 | const { fps: inputFPS } = await video.probeMaxQualityFile() | 98 | const { fps: inputFPS } = await video.probeMaxQualityFile() |
98 | const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) | 99 | const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) |
99 | const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) | 100 | const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) |
100 | 101 | ||
101 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) | 102 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) |
102 | 103 | ||
@@ -121,7 +122,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
121 | isNewVideo, | 122 | isNewVideo, |
122 | deleteWebVideoFiles: false, | 123 | deleteWebVideoFiles: false, |
123 | dependsOnRunnerJob, | 124 | dependsOnRunnerJob, |
124 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | 125 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) |
125 | }) | 126 | }) |
126 | continue | 127 | continue |
127 | } | 128 | } |
@@ -133,7 +134,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
133 | fps, | 134 | fps, |
134 | isNewVideo, | 135 | isNewVideo, |
135 | dependsOnRunnerJob, | 136 | dependsOnRunnerJob, |
136 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | 137 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) |
137 | }) | 138 | }) |
138 | continue | 139 | continue |
139 | } | 140 | } |
@@ -172,7 +173,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
172 | fps, | 173 | fps, |
173 | isNewVideo, | 174 | isNewVideo, |
174 | dependsOnRunnerJob: mainRunnerJob, | 175 | dependsOnRunnerJob: mainRunnerJob, |
175 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | 176 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) |
176 | }) | 177 | }) |
177 | } | 178 | } |
178 | 179 | ||
@@ -184,7 +185,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
184 | isNewVideo, | 185 | isNewVideo, |
185 | deleteWebVideoFiles: false, | 186 | deleteWebVideoFiles: false, |
186 | dependsOnRunnerJob: mainRunnerJob, | 187 | dependsOnRunnerJob: mainRunnerJob, |
187 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | 188 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) |
188 | }) | 189 | }) |
189 | } | 190 | } |
190 | } | 191 | } |
diff --git a/server/lib/transcoding/transcoding-priority.ts b/server/lib/transcoding/transcoding-priority.ts new file mode 100644 index 000000000..82ab6f2f1 --- /dev/null +++ b/server/lib/transcoding/transcoding-priority.ts | |||
@@ -0,0 +1,24 @@ | |||
1 | import { JOB_PRIORITY } from '@server/initializers/constants' | ||
2 | import { VideoModel } from '@server/models/video/video' | ||
3 | import { MUserId } from '@server/types/models' | ||
4 | |||
5 | export async function getTranscodingJobPriority (options: { | ||
6 | user: MUserId | ||
7 | fallback: number | ||
8 | type: 'vod' | 'studio' | ||
9 | }) { | ||
10 | const { user, fallback, type } = options | ||
11 | |||
12 | if (!user) return fallback | ||
13 | |||
14 | const now = new Date() | ||
15 | const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) | ||
16 | |||
17 | const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) | ||
18 | |||
19 | const base = type === 'vod' | ||
20 | ? JOB_PRIORITY.TRANSCODING | ||
21 | : JOB_PRIORITY.VIDEO_STUDIO | ||
22 | |||
23 | return base + videoUploadedByUser | ||
24 | } | ||
diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts index beda326a0..2c993faeb 100644 --- a/server/lib/video-studio.ts +++ b/server/lib/video-studio.ts | |||
@@ -1,19 +1,38 @@ | |||
1 | import { logger } from '@server/helpers/logger' | 1 | import { move, remove } from 'fs-extra' |
2 | import { MVideoFullLight } from '@server/types/models' | 2 | import { join } from 'path' |
3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
4 | import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' | ||
5 | import { CONFIG } from '@server/initializers/config' | ||
6 | import { UserModel } from '@server/models/user/user' | ||
7 | import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models' | ||
3 | import { getVideoStreamDuration } from '@shared/ffmpeg' | 8 | import { getVideoStreamDuration } from '@shared/ffmpeg' |
4 | import { VideoStudioEditionPayload, VideoStudioTask } from '@shared/models' | 9 | import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@shared/models' |
5 | import { remove } from 'fs-extra' | 10 | import { federateVideoIfNeeded } from './activitypub/videos' |
11 | import { JobQueue } from './job-queue' | ||
12 | import { VideoEditionTranscodingJobHandler } from './runners' | ||
13 | import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job' | ||
14 | import { getTranscodingJobPriority } from './transcoding/transcoding-priority' | ||
15 | import { buildNewFile, removeHLSPlaylist, removeWebTorrentFile } from './video-file' | ||
16 | import { VideoPathManager } from './video-path-manager' | ||
6 | 17 | ||
7 | function buildTaskFileFieldname (indice: number, fieldName = 'file') { | 18 | const lTags = loggerTagsFactory('video-edition') |
19 | |||
20 | export function buildTaskFileFieldname (indice: number, fieldName = 'file') { | ||
8 | return `tasks[${indice}][options][${fieldName}]` | 21 | return `tasks[${indice}][options][${fieldName}]` |
9 | } | 22 | } |
10 | 23 | ||
11 | function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { | 24 | export function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { |
12 | return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) | 25 | return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) |
13 | } | 26 | } |
14 | 27 | ||
15 | async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) { | 28 | export function getStudioTaskFilePath (filename: string) { |
16 | for (const task of payload.tasks) { | 29 | return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, filename) |
30 | } | ||
31 | |||
32 | export async function safeCleanupStudioTMPFiles (tasks: VideoStudioTaskPayload[]) { | ||
33 | logger.info('Removing studio task files', { tasks, ...lTags() }) | ||
34 | |||
35 | for (const task of tasks) { | ||
17 | try { | 36 | try { |
18 | if (task.name === 'add-intro' || task.name === 'add-outro') { | 37 | if (task.name === 'add-intro' || task.name === 'add-outro') { |
19 | await remove(task.options.file) | 38 | await remove(task.options.file) |
@@ -26,7 +45,13 @@ async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) { | |||
26 | } | 45 | } |
27 | } | 46 | } |
28 | 47 | ||
29 | async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) { | 48 | // --------------------------------------------------------------------------- |
49 | |||
50 | export async function approximateIntroOutroAdditionalSize ( | ||
51 | video: MVideoFullLight, | ||
52 | tasks: VideoStudioTask[], | ||
53 | fileFinder: (i: number) => string | ||
54 | ) { | ||
30 | let additionalDuration = 0 | 55 | let additionalDuration = 0 |
31 | 56 | ||
32 | for (let i = 0; i < tasks.length; i++) { | 57 | for (let i = 0; i < tasks.length; i++) { |
@@ -41,9 +66,65 @@ async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, task | |||
41 | return (video.getMaxQualityFile().size / video.duration) * additionalDuration | 66 | return (video.getMaxQualityFile().size / video.duration) * additionalDuration |
42 | } | 67 | } |
43 | 68 | ||
44 | export { | 69 | // --------------------------------------------------------------------------- |
45 | approximateIntroOutroAdditionalSize, | 70 | |
46 | buildTaskFileFieldname, | 71 | export async function createVideoStudioJob (options: { |
47 | getTaskFileFromReq, | 72 | video: MVideo |
48 | safeCleanupStudioTMPFiles | 73 | user: MUser |
74 | payload: VideoStudioEditionPayload | ||
75 | }) { | ||
76 | const { video, user, payload } = options | ||
77 | |||
78 | const priority = await getTranscodingJobPriority({ user, type: 'studio', fallback: 0 }) | ||
79 | |||
80 | if (CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED) { | ||
81 | await new VideoEditionTranscodingJobHandler().create({ video, tasks: payload.tasks, priority }) | ||
82 | return | ||
83 | } | ||
84 | |||
85 | await JobQueue.Instance.createJob({ type: 'video-studio-edition', payload, priority }) | ||
86 | } | ||
87 | |||
88 | export async function onVideoEditionEnded (options: { | ||
89 | editionResultPath: string | ||
90 | tasks: VideoStudioTaskPayload[] | ||
91 | video: MVideoFullLight | ||
92 | }) { | ||
93 | const { video, tasks, editionResultPath } = options | ||
94 | |||
95 | const newFile = await buildNewFile({ path: editionResultPath, mode: 'web-video' }) | ||
96 | newFile.videoId = video.id | ||
97 | |||
98 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) | ||
99 | await move(editionResultPath, outputPath) | ||
100 | |||
101 | await safeCleanupStudioTMPFiles(tasks) | ||
102 | |||
103 | await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) | ||
104 | await removeAllFiles(video, newFile) | ||
105 | |||
106 | await newFile.save() | ||
107 | |||
108 | video.duration = await getVideoStreamDuration(outputPath) | ||
109 | await video.save() | ||
110 | |||
111 | await federateVideoIfNeeded(video, false, undefined) | ||
112 | |||
113 | const user = await UserModel.loadByVideoId(video.id) | ||
114 | |||
115 | await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) | ||
116 | } | ||
117 | |||
118 | // --------------------------------------------------------------------------- | ||
119 | // Private | ||
120 | // --------------------------------------------------------------------------- | ||
121 | |||
122 | async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) { | ||
123 | await removeHLSPlaylist(video) | ||
124 | |||
125 | for (const file of video.VideoFiles) { | ||
126 | if (file.id === webTorrentFileException.id) continue | ||
127 | |||
128 | await removeWebTorrentFile(video, file.id) | ||
129 | } | ||
49 | } | 130 | } |
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index b3e7e5011..a0074cb24 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts | |||
@@ -62,6 +62,7 @@ const customConfigUpdateValidator = [ | |||
62 | body('transcoding.hls.enabled').isBoolean(), | 62 | body('transcoding.hls.enabled').isBoolean(), |
63 | 63 | ||
64 | body('videoStudio.enabled').isBoolean(), | 64 | body('videoStudio.enabled').isBoolean(), |
65 | body('videoStudio.remoteRunners.enabled').isBoolean(), | ||
65 | 66 | ||
66 | body('import.videos.concurrency').isInt({ min: 0 }), | 67 | body('import.videos.concurrency').isInt({ min: 0 }), |
67 | body('import.videos.http.enabled').isBoolean(), | 68 | body('import.videos.http.enabled').isBoolean(), |
diff --git a/server/middlewares/validators/runners/job-files.ts b/server/middlewares/validators/runners/job-files.ts index 56afa39aa..e5afff0e5 100644 --- a/server/middlewares/validators/runners/job-files.ts +++ b/server/middlewares/validators/runners/job-files.ts | |||
@@ -1,5 +1,8 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { HttpStatusCode } from '@shared/models' | 2 | import { param } from 'express-validator' |
3 | import { basename } from 'path' | ||
4 | import { isSafeFilename } from '@server/helpers/custom-validators/misc' | ||
5 | import { hasVideoStudioTaskFile, HttpStatusCode, RunnerJobVideoEditionTranscodingPayload } from '@shared/models' | ||
3 | import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' | 6 | import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' |
4 | 7 | ||
5 | const tags = [ 'runner' ] | 8 | const tags = [ 'runner' ] |
@@ -25,3 +28,33 @@ export const runnerJobGetVideoTranscodingFileValidator = [ | |||
25 | return next() | 28 | return next() |
26 | } | 29 | } |
27 | ] | 30 | ] |
31 | |||
32 | export const runnerJobGetVideoStudioTaskFileValidator = [ | ||
33 | param('filename').custom(v => isSafeFilename(v)), | ||
34 | |||
35 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
36 | if (areValidationErrors(req, res)) return | ||
37 | |||
38 | const filename = req.params.filename | ||
39 | |||
40 | const payload = res.locals.runnerJob.payload as RunnerJobVideoEditionTranscodingPayload | ||
41 | |||
42 | const found = Array.isArray(payload?.tasks) && payload.tasks.some(t => { | ||
43 | if (hasVideoStudioTaskFile(t)) { | ||
44 | return basename(t.options.file) === filename | ||
45 | } | ||
46 | |||
47 | return false | ||
48 | }) | ||
49 | |||
50 | if (!found) { | ||
51 | return res.fail({ | ||
52 | status: HttpStatusCode.BAD_REQUEST_400, | ||
53 | message: 'File is not associated to this edition task', | ||
54 | tags: [ ...tags, res.locals.videoAll.uuid ] | ||
55 | }) | ||
56 | } | ||
57 | |||
58 | return next() | ||
59 | } | ||
60 | ] | ||
diff --git a/server/middlewares/validators/runners/jobs.ts b/server/middlewares/validators/runners/jobs.ts index 8cb87e946..de956a1ca 100644 --- a/server/middlewares/validators/runners/jobs.ts +++ b/server/middlewares/validators/runners/jobs.ts | |||
@@ -91,6 +91,28 @@ export const successRunnerJobValidator = [ | |||
91 | } | 91 | } |
92 | ] | 92 | ] |
93 | 93 | ||
94 | export const cancelRunnerJobValidator = [ | ||
95 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
96 | const runnerJob = res.locals.runnerJob | ||
97 | |||
98 | const allowedStates = new Set<RunnerJobState>([ | ||
99 | RunnerJobState.PENDING, | ||
100 | RunnerJobState.PROCESSING, | ||
101 | RunnerJobState.WAITING_FOR_PARENT_JOB | ||
102 | ]) | ||
103 | |||
104 | if (allowedStates.has(runnerJob.state) !== true) { | ||
105 | return res.fail({ | ||
106 | status: HttpStatusCode.BAD_REQUEST_400, | ||
107 | message: 'Cannot cancel this job that is not in "pending", "processing" or "waiting for parent job" state', | ||
108 | tags | ||
109 | }) | ||
110 | } | ||
111 | |||
112 | return next() | ||
113 | } | ||
114 | ] | ||
115 | |||
94 | export const runnerJobGetValidator = [ | 116 | export const runnerJobGetValidator = [ |
95 | param('jobUUID').custom(isUUIDValid), | 117 | param('jobUUID').custom(isUUIDValid), |
96 | 118 | ||
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index c5cda203e..472cad182 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts | |||
@@ -162,7 +162,10 @@ describe('Test config API validators', function () { | |||
162 | } | 162 | } |
163 | }, | 163 | }, |
164 | videoStudio: { | 164 | videoStudio: { |
165 | enabled: true | 165 | enabled: true, |
166 | remoteRunners: { | ||
167 | enabled: true | ||
168 | } | ||
166 | }, | 169 | }, |
167 | import: { | 170 | import: { |
168 | videos: { | 171 | videos: { |
diff --git a/server/tests/api/check-params/runners.ts b/server/tests/api/check-params/runners.ts index 4da6fd91d..90a301392 100644 --- a/server/tests/api/check-params/runners.ts +++ b/server/tests/api/check-params/runners.ts | |||
@@ -1,6 +1,17 @@ | |||
1 | import { basename } from 'path' | ||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 2 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' | 3 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' |
3 | import { HttpStatusCode, RunnerJob, RunnerJobState, RunnerJobSuccessPayload, RunnerJobUpdatePayload, VideoPrivacy } from '@shared/models' | 4 | import { |
5 | HttpStatusCode, | ||
6 | isVideoStudioTaskIntro, | ||
7 | RunnerJob, | ||
8 | RunnerJobState, | ||
9 | RunnerJobSuccessPayload, | ||
10 | RunnerJobUpdatePayload, | ||
11 | RunnerJobVideoEditionTranscodingPayload, | ||
12 | VideoPrivacy, | ||
13 | VideoStudioTaskIntro | ||
14 | } from '@shared/models' | ||
4 | import { | 15 | import { |
5 | cleanupTests, | 16 | cleanupTests, |
6 | createSingleServer, | 17 | createSingleServer, |
@@ -10,6 +21,7 @@ import { | |||
10 | setAccessTokensToServers, | 21 | setAccessTokensToServers, |
11 | setDefaultVideoChannel, | 22 | setDefaultVideoChannel, |
12 | stopFfmpeg, | 23 | stopFfmpeg, |
24 | VideoStudioCommand, | ||
13 | waitJobs | 25 | waitJobs |
14 | } from '@shared/server-commands' | 26 | } from '@shared/server-commands' |
15 | 27 | ||
@@ -53,7 +65,10 @@ describe('Test managing runners', function () { | |||
53 | registrationTokenId = data[0].id | 65 | registrationTokenId = data[0].id |
54 | 66 | ||
55 | await server.config.enableTranscoding(true, true) | 67 | await server.config.enableTranscoding(true, true) |
68 | await server.config.enableStudio() | ||
56 | await server.config.enableRemoteTranscoding() | 69 | await server.config.enableRemoteTranscoding() |
70 | await server.config.enableRemoteStudio() | ||
71 | |||
57 | runnerToken = await server.runners.autoRegisterRunner() | 72 | runnerToken = await server.runners.autoRegisterRunner() |
58 | runnerToken2 = await server.runners.autoRegisterRunner() | 73 | runnerToken2 = await server.runners.autoRegisterRunner() |
59 | 74 | ||
@@ -249,6 +264,10 @@ describe('Test managing runners', function () { | |||
249 | await server.runnerJobs.cancelByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 264 | await server.runnerJobs.cancelByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
250 | }) | 265 | }) |
251 | 266 | ||
267 | it('Should fail with an already cancelled job', async function () { | ||
268 | await server.runnerJobs.cancelByAdmin({ jobUUID: cancelledJobUUID, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | ||
269 | }) | ||
270 | |||
252 | it('Should succeed with the correct params', async function () { | 271 | it('Should succeed with the correct params', async function () { |
253 | await server.runnerJobs.cancelByAdmin({ jobUUID }) | 272 | await server.runnerJobs.cancelByAdmin({ jobUUID }) |
254 | }) | 273 | }) |
@@ -296,9 +315,13 @@ describe('Test managing runners', function () { | |||
296 | 315 | ||
297 | let pendingUUID: string | 316 | let pendingUUID: string |
298 | 317 | ||
318 | let videoStudioUUID: string | ||
319 | let studioFile: string | ||
320 | |||
299 | let liveAcceptedJob: RunnerJob & { jobToken: string } | 321 | let liveAcceptedJob: RunnerJob & { jobToken: string } |
322 | let studioAcceptedJob: RunnerJob & { jobToken: string } | ||
300 | 323 | ||
301 | async function fetchFiles (options: { | 324 | async function fetchVideoInputFiles (options: { |
302 | jobUUID: string | 325 | jobUUID: string |
303 | videoUUID: string | 326 | videoUUID: string |
304 | runnerToken: string | 327 | runnerToken: string |
@@ -315,6 +338,21 @@ describe('Test managing runners', function () { | |||
315 | } | 338 | } |
316 | } | 339 | } |
317 | 340 | ||
341 | async function fetchStudioFiles (options: { | ||
342 | jobUUID: string | ||
343 | videoUUID: string | ||
344 | runnerToken: string | ||
345 | jobToken: string | ||
346 | studioFile?: string | ||
347 | expectedStatus: HttpStatusCode | ||
348 | }) { | ||
349 | const { jobUUID, expectedStatus, videoUUID, runnerToken, jobToken, studioFile } = options | ||
350 | |||
351 | const path = `/api/v1/runners/jobs/${jobUUID}/files/videos/${videoUUID}/studio/task-files/${studioFile}` | ||
352 | |||
353 | await makePostBodyRequest({ url: server.url, path, fields: { runnerToken, jobToken }, expectedStatus }) | ||
354 | } | ||
355 | |||
318 | before(async function () { | 356 | before(async function () { |
319 | this.timeout(120000) | 357 | this.timeout(120000) |
320 | 358 | ||
@@ -353,6 +391,28 @@ describe('Test managing runners', function () { | |||
353 | } | 391 | } |
354 | 392 | ||
355 | { | 393 | { |
394 | await server.config.disableTranscoding() | ||
395 | |||
396 | const { uuid } = await server.videos.quickUpload({ name: 'video studio' }) | ||
397 | videoStudioUUID = uuid | ||
398 | |||
399 | await server.config.enableTranscoding(true, true) | ||
400 | await server.config.enableStudio() | ||
401 | |||
402 | await server.videoStudio.createEditionTasks({ | ||
403 | videoId: videoStudioUUID, | ||
404 | tasks: VideoStudioCommand.getComplexTask() | ||
405 | }) | ||
406 | |||
407 | const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'video-edition-transcoding' }) | ||
408 | studioAcceptedJob = job | ||
409 | |||
410 | const tasks = (job.payload as RunnerJobVideoEditionTranscodingPayload).tasks | ||
411 | const fileUrl = (tasks.find(t => isVideoStudioTaskIntro(t)) as VideoStudioTaskIntro).options.file as string | ||
412 | studioFile = basename(fileUrl) | ||
413 | } | ||
414 | |||
415 | { | ||
356 | await server.config.enableLive({ | 416 | await server.config.enableLive({ |
357 | allowReplay: false, | 417 | allowReplay: false, |
358 | resolutions: 'max', | 418 | resolutions: 'max', |
@@ -381,8 +441,6 @@ describe('Test managing runners', function () { | |||
381 | jobToken: string | 441 | jobToken: string |
382 | expectedStatus: HttpStatusCode | 442 | expectedStatus: HttpStatusCode |
383 | }) { | 443 | }) { |
384 | await fetchFiles({ ...options, videoUUID }) | ||
385 | |||
386 | await server.runnerJobs.abort({ ...options, reason: 'reason' }) | 444 | await server.runnerJobs.abort({ ...options, reason: 'reason' }) |
387 | await server.runnerJobs.update({ ...options }) | 445 | await server.runnerJobs.update({ ...options }) |
388 | await server.runnerJobs.error({ ...options, message: 'message' }) | 446 | await server.runnerJobs.error({ ...options, message: 'message' }) |
@@ -390,39 +448,95 @@ describe('Test managing runners', function () { | |||
390 | } | 448 | } |
391 | 449 | ||
392 | it('Should fail with an invalid job uuid', async function () { | 450 | it('Should fail with an invalid job uuid', async function () { |
393 | await testEndpoints({ jobUUID: 'a', runnerToken, jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 451 | const options = { jobUUID: 'a', runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } |
452 | |||
453 | await testEndpoints({ ...options, jobToken }) | ||
454 | await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) | ||
455 | await fetchStudioFiles({ ...options, videoUUID, jobToken: studioAcceptedJob.jobToken, studioFile }) | ||
394 | }) | 456 | }) |
395 | 457 | ||
396 | it('Should fail with an unknown job uuid', async function () { | 458 | it('Should fail with an unknown job uuid', async function () { |
397 | const jobUUID = badUUID | 459 | const options = { jobUUID: badUUID, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } |
398 | await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 460 | |
461 | await testEndpoints({ ...options, jobToken }) | ||
462 | await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) | ||
463 | await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID, studioFile }) | ||
399 | }) | 464 | }) |
400 | 465 | ||
401 | it('Should fail with an invalid runner token', async function () { | 466 | it('Should fail with an invalid runner token', async function () { |
402 | await testEndpoints({ jobUUID, runnerToken: '', jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 467 | const options = { runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } |
468 | |||
469 | await testEndpoints({ ...options, jobUUID, jobToken }) | ||
470 | await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) | ||
471 | await fetchStudioFiles({ | ||
472 | ...options, | ||
473 | jobToken: studioAcceptedJob.jobToken, | ||
474 | jobUUID: studioAcceptedJob.uuid, | ||
475 | videoUUID: videoStudioUUID, | ||
476 | studioFile | ||
477 | }) | ||
403 | }) | 478 | }) |
404 | 479 | ||
405 | it('Should fail with an unknown runner token', async function () { | 480 | it('Should fail with an unknown runner token', async function () { |
406 | const runnerToken = badUUID | 481 | const options = { runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } |
407 | await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 482 | |
483 | await testEndpoints({ ...options, jobUUID, jobToken }) | ||
484 | await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) | ||
485 | await fetchStudioFiles({ | ||
486 | ...options, | ||
487 | jobToken: studioAcceptedJob.jobToken, | ||
488 | jobUUID: studioAcceptedJob.uuid, | ||
489 | videoUUID: videoStudioUUID, | ||
490 | studioFile | ||
491 | }) | ||
408 | }) | 492 | }) |
409 | 493 | ||
410 | it('Should fail with an invalid job token job uuid', async function () { | 494 | it('Should fail with an invalid job token job uuid', async function () { |
411 | await testEndpoints({ jobUUID, runnerToken, jobToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 495 | const options = { runnerToken, jobToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } |
496 | |||
497 | await testEndpoints({ ...options, jobUUID }) | ||
498 | await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) | ||
499 | await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) | ||
412 | }) | 500 | }) |
413 | 501 | ||
414 | it('Should fail with an unknown job token job uuid', async function () { | 502 | it('Should fail with an unknown job token job uuid', async function () { |
415 | const jobToken = badUUID | 503 | const options = { runnerToken, jobToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } |
416 | await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 504 | |
505 | await testEndpoints({ ...options, jobUUID }) | ||
506 | await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) | ||
507 | await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) | ||
417 | }) | 508 | }) |
418 | 509 | ||
419 | it('Should fail with a runner token not associated to this job', async function () { | 510 | it('Should fail with a runner token not associated to this job', async function () { |
420 | await testEndpoints({ jobUUID, runnerToken: runnerToken2, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 511 | const options = { runnerToken: runnerToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } |
512 | |||
513 | await testEndpoints({ ...options, jobUUID, jobToken }) | ||
514 | await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) | ||
515 | await fetchStudioFiles({ | ||
516 | ...options, | ||
517 | jobToken: studioAcceptedJob.jobToken, | ||
518 | jobUUID: studioAcceptedJob.uuid, | ||
519 | videoUUID: videoStudioUUID, | ||
520 | studioFile | ||
521 | }) | ||
421 | }) | 522 | }) |
422 | 523 | ||
423 | it('Should fail with a job uuid not associated to the job token', async function () { | 524 | it('Should fail with a job uuid not associated to the job token', async function () { |
424 | await testEndpoints({ jobUUID: jobUUID2, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 525 | { |
425 | await testEndpoints({ jobUUID, runnerToken, jobToken: jobToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 526 | const options = { jobUUID: jobUUID2, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } |
527 | |||
528 | await testEndpoints({ ...options, jobToken }) | ||
529 | await fetchVideoInputFiles({ ...options, jobToken, videoUUID }) | ||
530 | await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID: videoStudioUUID, studioFile }) | ||
531 | } | ||
532 | |||
533 | { | ||
534 | const options = { runnerToken, jobToken: jobToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } | ||
535 | |||
536 | await testEndpoints({ ...options, jobUUID }) | ||
537 | await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) | ||
538 | await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) | ||
539 | } | ||
426 | }) | 540 | }) |
427 | }) | 541 | }) |
428 | 542 | ||
@@ -670,27 +784,82 @@ describe('Test managing runners', function () { | |||
670 | }) | 784 | }) |
671 | }) | 785 | }) |
672 | }) | 786 | }) |
787 | |||
788 | describe('Video studio', function () { | ||
789 | |||
790 | it('Should fail with an invalid video edition transcoding payload', async function () { | ||
791 | await server.runnerJobs.success({ | ||
792 | jobUUID: studioAcceptedJob.uuid, | ||
793 | jobToken: studioAcceptedJob.jobToken, | ||
794 | payload: { hello: 'video_short.mp4' } as any, | ||
795 | runnerToken, | ||
796 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
797 | }) | ||
798 | }) | ||
799 | }) | ||
673 | }) | 800 | }) |
674 | 801 | ||
675 | describe('Job files', function () { | 802 | describe('Job files', function () { |
676 | 803 | ||
677 | describe('Video files', function () { | 804 | describe('Check video param for common job file routes', function () { |
805 | |||
806 | async function fetchFiles (options: { | ||
807 | videoUUID?: string | ||
808 | expectedStatus: HttpStatusCode | ||
809 | }) { | ||
810 | await fetchVideoInputFiles({ videoUUID, ...options, jobToken, jobUUID, runnerToken }) | ||
811 | |||
812 | await fetchStudioFiles({ | ||
813 | videoUUID: videoStudioUUID, | ||
814 | |||
815 | ...options, | ||
816 | |||
817 | jobToken: studioAcceptedJob.jobToken, | ||
818 | jobUUID: studioAcceptedJob.uuid, | ||
819 | runnerToken, | ||
820 | studioFile | ||
821 | }) | ||
822 | } | ||
678 | 823 | ||
679 | it('Should fail with an invalid video id', async function () { | 824 | it('Should fail with an invalid video id', async function () { |
680 | await fetchFiles({ videoUUID: 'a', jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 825 | await fetchFiles({ |
826 | videoUUID: 'a', | ||
827 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
828 | }) | ||
681 | }) | 829 | }) |
682 | 830 | ||
683 | it('Should fail with an unknown video id', async function () { | 831 | it('Should fail with an unknown video id', async function () { |
684 | const videoUUID = '910ec12a-d9e6-458b-a274-0abb655f9464' | 832 | const videoUUID = '910ec12a-d9e6-458b-a274-0abb655f9464' |
685 | await fetchFiles({ videoUUID, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 833 | |
834 | await fetchFiles({ | ||
835 | videoUUID, | ||
836 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
837 | }) | ||
686 | }) | 838 | }) |
687 | 839 | ||
688 | it('Should fail with a video id not associated to this job', async function () { | 840 | it('Should fail with a video id not associated to this job', async function () { |
689 | await fetchFiles({ videoUUID: videoUUID2, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | 841 | await fetchFiles({ |
842 | videoUUID: videoUUID2, | ||
843 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
844 | }) | ||
690 | }) | 845 | }) |
691 | 846 | ||
692 | it('Should succeed with the correct params', async function () { | 847 | it('Should succeed with the correct params', async function () { |
693 | await fetchFiles({ videoUUID, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.OK_200 }) | 848 | await fetchFiles({ expectedStatus: HttpStatusCode.OK_200 }) |
849 | }) | ||
850 | }) | ||
851 | |||
852 | describe('Video edition tasks file routes', function () { | ||
853 | |||
854 | it('Should fail with an invalid studio filename', async function () { | ||
855 | await fetchStudioFiles({ | ||
856 | videoUUID: videoStudioUUID, | ||
857 | jobUUID: studioAcceptedJob.uuid, | ||
858 | runnerToken, | ||
859 | jobToken: studioAcceptedJob.jobToken, | ||
860 | studioFile: 'toto', | ||
861 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
862 | }) | ||
694 | }) | 863 | }) |
695 | }) | 864 | }) |
696 | }) | 865 | }) |
diff --git a/server/tests/api/runners/index.ts b/server/tests/api/runners/index.ts index 7f33ec8dd..642a3a96d 100644 --- a/server/tests/api/runners/index.ts +++ b/server/tests/api/runners/index.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | export * from './runner-common' | 1 | export * from './runner-common' |
2 | export * from './runner-live-transcoding' | 2 | export * from './runner-live-transcoding' |
3 | export * from './runner-socket' | 3 | export * from './runner-socket' |
4 | export * from './runner-studio-transcoding' | ||
4 | export * from './runner-vod-transcoding' | 5 | export * from './runner-vod-transcoding' |
diff --git a/server/tests/api/runners/runner-common.ts b/server/tests/api/runners/runner-common.ts index a2204753b..554024190 100644 --- a/server/tests/api/runners/runner-common.ts +++ b/server/tests/api/runners/runner-common.ts | |||
@@ -2,7 +2,15 @@ | |||
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { wait } from '@shared/core-utils' | 4 | import { wait } from '@shared/core-utils' |
5 | import { HttpStatusCode, Runner, RunnerJob, RunnerJobAdmin, RunnerJobState, RunnerRegistrationToken } from '@shared/models' | 5 | import { |
6 | HttpStatusCode, | ||
7 | Runner, | ||
8 | RunnerJob, | ||
9 | RunnerJobAdmin, | ||
10 | RunnerJobState, | ||
11 | RunnerJobVODWebVideoTranscodingPayload, | ||
12 | RunnerRegistrationToken | ||
13 | } from '@shared/models' | ||
6 | import { | 14 | import { |
7 | cleanupTests, | 15 | cleanupTests, |
8 | createSingleServer, | 16 | createSingleServer, |
@@ -349,7 +357,7 @@ describe('Test runner common actions', function () { | |||
349 | for (const job of availableJobs) { | 357 | for (const job of availableJobs) { |
350 | expect(job.uuid).to.exist | 358 | expect(job.uuid).to.exist |
351 | expect(job.payload.input).to.exist | 359 | expect(job.payload.input).to.exist |
352 | expect(job.payload.output).to.exist | 360 | expect((job.payload as RunnerJobVODWebVideoTranscodingPayload).output).to.exist |
353 | 361 | ||
354 | expect((job as RunnerJobAdmin).privatePayload).to.not.exist | 362 | expect((job as RunnerJobAdmin).privatePayload).to.not.exist |
355 | } | 363 | } |
diff --git a/server/tests/api/runners/runner-studio-transcoding.ts b/server/tests/api/runners/runner-studio-transcoding.ts new file mode 100644 index 000000000..9ae629be6 --- /dev/null +++ b/server/tests/api/runners/runner-studio-transcoding.ts | |||
@@ -0,0 +1,168 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { readFile } from 'fs-extra' | ||
5 | import { checkPersistentTmpIsEmpty, checkVideoDuration } from '@server/tests/shared' | ||
6 | import { buildAbsoluteFixturePath } from '@shared/core-utils' | ||
7 | import { | ||
8 | RunnerJobVideoEditionTranscodingPayload, | ||
9 | VideoEditionTranscodingSuccess, | ||
10 | VideoState, | ||
11 | VideoStudioTask, | ||
12 | VideoStudioTaskIntro | ||
13 | } from '@shared/models' | ||
14 | import { | ||
15 | cleanupTests, | ||
16 | createMultipleServers, | ||
17 | doubleFollow, | ||
18 | PeerTubeServer, | ||
19 | setAccessTokensToServers, | ||
20 | setDefaultVideoChannel, | ||
21 | VideoStudioCommand, | ||
22 | waitJobs | ||
23 | } from '@shared/server-commands' | ||
24 | |||
25 | describe('Test runner video studio transcoding', function () { | ||
26 | let servers: PeerTubeServer[] = [] | ||
27 | let runnerToken: string | ||
28 | let videoUUID: string | ||
29 | let jobUUID: string | ||
30 | |||
31 | async function renewStudio (tasks: VideoStudioTask[] = VideoStudioCommand.getComplexTask()) { | ||
32 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) | ||
33 | videoUUID = uuid | ||
34 | |||
35 | await waitJobs(servers) | ||
36 | |||
37 | await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks }) | ||
38 | await waitJobs(servers) | ||
39 | |||
40 | const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) | ||
41 | expect(availableJobs).to.have.lengthOf(1) | ||
42 | |||
43 | jobUUID = availableJobs[0].uuid | ||
44 | } | ||
45 | |||
46 | before(async function () { | ||
47 | this.timeout(120_000) | ||
48 | |||
49 | servers = await createMultipleServers(2) | ||
50 | |||
51 | await setAccessTokensToServers(servers) | ||
52 | await setDefaultVideoChannel(servers) | ||
53 | |||
54 | await doubleFollow(servers[0], servers[1]) | ||
55 | |||
56 | await servers[0].config.enableTranscoding(true, true) | ||
57 | await servers[0].config.enableStudio() | ||
58 | await servers[0].config.enableRemoteStudio() | ||
59 | |||
60 | runnerToken = await servers[0].runners.autoRegisterRunner() | ||
61 | }) | ||
62 | |||
63 | it('Should error a studio transcoding job', async function () { | ||
64 | this.timeout(60000) | ||
65 | |||
66 | await renewStudio() | ||
67 | |||
68 | for (let i = 0; i < 5; i++) { | ||
69 | const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) | ||
70 | const jobToken = job.jobToken | ||
71 | |||
72 | await servers[0].runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) | ||
73 | } | ||
74 | |||
75 | const video = await servers[0].videos.get({ id: videoUUID }) | ||
76 | expect(video.state.id).to.equal(VideoState.PUBLISHED) | ||
77 | |||
78 | await checkPersistentTmpIsEmpty(servers[0]) | ||
79 | }) | ||
80 | |||
81 | it('Should cancel a transcoding job', async function () { | ||
82 | this.timeout(60000) | ||
83 | |||
84 | await renewStudio() | ||
85 | |||
86 | await servers[0].runnerJobs.cancelByAdmin({ jobUUID }) | ||
87 | |||
88 | const video = await servers[0].videos.get({ id: videoUUID }) | ||
89 | expect(video.state.id).to.equal(VideoState.PUBLISHED) | ||
90 | |||
91 | await checkPersistentTmpIsEmpty(servers[0]) | ||
92 | }) | ||
93 | |||
94 | it('Should execute a remote studio job', async function () { | ||
95 | this.timeout(240_000) | ||
96 | |||
97 | const tasks = [ | ||
98 | { | ||
99 | name: 'add-outro' as 'add-outro', | ||
100 | options: { | ||
101 | file: 'video_short.webm' | ||
102 | } | ||
103 | }, | ||
104 | { | ||
105 | name: 'add-watermark' as 'add-watermark', | ||
106 | options: { | ||
107 | file: 'thumbnail.png' | ||
108 | } | ||
109 | }, | ||
110 | { | ||
111 | name: 'add-intro' as 'add-intro', | ||
112 | options: { | ||
113 | file: 'video_very_short_240p.mp4' | ||
114 | } | ||
115 | } | ||
116 | ] | ||
117 | |||
118 | await renewStudio(tasks) | ||
119 | |||
120 | for (const server of servers) { | ||
121 | await checkVideoDuration(server, videoUUID, 5) | ||
122 | } | ||
123 | |||
124 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVideoEditionTranscodingPayload>({ runnerToken, jobUUID }) | ||
125 | const jobToken = job.jobToken | ||
126 | |||
127 | expect(job.type === 'video-edition-transcoding') | ||
128 | expect(job.payload.input.videoFileUrl).to.exist | ||
129 | |||
130 | // Check video input file | ||
131 | { | ||
132 | await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) | ||
133 | } | ||
134 | |||
135 | // Check task files | ||
136 | for (let i = 0; i < tasks.length; i++) { | ||
137 | const task = tasks[i] | ||
138 | const payloadTask = job.payload.tasks[i] | ||
139 | |||
140 | expect(payloadTask.name).to.equal(task.name) | ||
141 | |||
142 | const inputFile = await readFile(buildAbsoluteFixturePath(task.options.file)) | ||
143 | |||
144 | const { body } = await servers[0].runnerJobs.getJobFile({ | ||
145 | url: (payloadTask as VideoStudioTaskIntro).options.file as string, | ||
146 | jobToken, | ||
147 | runnerToken | ||
148 | }) | ||
149 | |||
150 | expect(body).to.deep.equal(inputFile) | ||
151 | } | ||
152 | |||
153 | const payload: VideoEditionTranscodingSuccess = { videoFile: 'video_very_short_240p.mp4' } | ||
154 | await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) | ||
155 | |||
156 | await waitJobs(servers) | ||
157 | |||
158 | for (const server of servers) { | ||
159 | await checkVideoDuration(server, videoUUID, 2) | ||
160 | } | ||
161 | |||
162 | await checkPersistentTmpIsEmpty(servers[0]) | ||
163 | }) | ||
164 | |||
165 | after(async function () { | ||
166 | await cleanupTests(servers) | ||
167 | }) | ||
168 | }) | ||
diff --git a/server/tests/api/runners/runner-vod-transcoding.ts b/server/tests/api/runners/runner-vod-transcoding.ts index 92a47ac3b..b08ee312c 100644 --- a/server/tests/api/runners/runner-vod-transcoding.ts +++ b/server/tests/api/runners/runner-vod-transcoding.ts | |||
@@ -155,7 +155,7 @@ describe('Test runner VOD transcoding', function () { | |||
155 | expect(job.payload.output.resolution).to.equal(720) | 155 | expect(job.payload.output.resolution).to.equal(720) |
156 | expect(job.payload.output.fps).to.equal(25) | 156 | expect(job.payload.output.fps).to.equal(25) |
157 | 157 | ||
158 | const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) | 158 | const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) |
159 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short.webm')) | 159 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short.webm')) |
160 | 160 | ||
161 | expect(body).to.deep.equal(inputFile) | 161 | expect(body).to.deep.equal(inputFile) |
@@ -200,7 +200,7 @@ describe('Test runner VOD transcoding', function () { | |||
200 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODWebVideoTranscodingPayload>({ runnerToken, jobUUID }) | 200 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODWebVideoTranscodingPayload>({ runnerToken, jobUUID }) |
201 | jobToken = job.jobToken | 201 | jobToken = job.jobToken |
202 | 202 | ||
203 | const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) | 203 | const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) |
204 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) | 204 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) |
205 | 205 | ||
206 | expect(body).to.deep.equal(inputFile) | 206 | expect(body).to.deep.equal(inputFile) |
@@ -221,7 +221,7 @@ describe('Test runner VOD transcoding', function () { | |||
221 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODWebVideoTranscodingPayload>({ runnerToken, jobUUID }) | 221 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODWebVideoTranscodingPayload>({ runnerToken, jobUUID }) |
222 | jobToken = job.jobToken | 222 | jobToken = job.jobToken |
223 | 223 | ||
224 | const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) | 224 | const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) |
225 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) | 225 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) |
226 | expect(body).to.deep.equal(inputFile) | 226 | expect(body).to.deep.equal(inputFile) |
227 | 227 | ||
@@ -293,7 +293,7 @@ describe('Test runner VOD transcoding', function () { | |||
293 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODHLSTranscodingPayload>({ runnerToken, jobUUID }) | 293 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODHLSTranscodingPayload>({ runnerToken, jobUUID }) |
294 | jobToken = job.jobToken | 294 | jobToken = job.jobToken |
295 | 295 | ||
296 | const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) | 296 | const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) |
297 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) | 297 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) |
298 | 298 | ||
299 | expect(body).to.deep.equal(inputFile) | 299 | expect(body).to.deep.equal(inputFile) |
@@ -337,7 +337,7 @@ describe('Test runner VOD transcoding', function () { | |||
337 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODHLSTranscodingPayload>({ runnerToken, jobUUID }) | 337 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODHLSTranscodingPayload>({ runnerToken, jobUUID }) |
338 | jobToken = job.jobToken | 338 | jobToken = job.jobToken |
339 | 339 | ||
340 | const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) | 340 | const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) |
341 | const inputFile = await readFile(buildAbsoluteFixturePath(maxQualityFile)) | 341 | const inputFile = await readFile(buildAbsoluteFixturePath(maxQualityFile)) |
342 | expect(body).to.deep.equal(inputFile) | 342 | expect(body).to.deep.equal(inputFile) |
343 | 343 | ||
@@ -446,13 +446,13 @@ describe('Test runner VOD transcoding', function () { | |||
446 | expect(job.payload.output.resolution).to.equal(480) | 446 | expect(job.payload.output.resolution).to.equal(480) |
447 | 447 | ||
448 | { | 448 | { |
449 | const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.audioFileUrl, jobToken, runnerToken }) | 449 | const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.audioFileUrl, jobToken, runnerToken }) |
450 | const inputFile = await readFile(buildAbsoluteFixturePath('sample.ogg')) | 450 | const inputFile = await readFile(buildAbsoluteFixturePath('sample.ogg')) |
451 | expect(body).to.deep.equal(inputFile) | 451 | expect(body).to.deep.equal(inputFile) |
452 | } | 452 | } |
453 | 453 | ||
454 | { | 454 | { |
455 | const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.previewFileUrl, jobToken, runnerToken }) | 455 | const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.previewFileUrl, jobToken, runnerToken }) |
456 | 456 | ||
457 | const video = await servers[0].videos.get({ id: videoUUID }) | 457 | const video = await servers[0].videos.get({ id: videoUUID }) |
458 | const { body: inputFile } = await makeGetRequest({ | 458 | const { body: inputFile } = await makeGetRequest({ |
@@ -503,7 +503,7 @@ describe('Test runner VOD transcoding', function () { | |||
503 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODHLSTranscodingPayload>({ runnerToken, jobUUID }) | 503 | const { job } = await servers[0].runnerJobs.accept<RunnerJobVODHLSTranscodingPayload>({ runnerToken, jobUUID }) |
504 | jobToken = job.jobToken | 504 | jobToken = job.jobToken |
505 | 505 | ||
506 | const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) | 506 | const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) |
507 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short_480p.mp4')) | 507 | const inputFile = await readFile(buildAbsoluteFixturePath('video_short_480p.mp4')) |
508 | expect(body).to.deep.equal(inputFile) | 508 | expect(body).to.deep.equal(inputFile) |
509 | 509 | ||
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 54a40b994..011ba268c 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts | |||
@@ -102,6 +102,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { | |||
102 | expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.true | 102 | expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.true |
103 | 103 | ||
104 | expect(data.videoStudio.enabled).to.be.false | 104 | expect(data.videoStudio.enabled).to.be.false |
105 | expect(data.videoStudio.remoteRunners.enabled).to.be.false | ||
105 | 106 | ||
106 | expect(data.import.videos.concurrency).to.equal(2) | 107 | expect(data.import.videos.concurrency).to.equal(2) |
107 | expect(data.import.videos.http.enabled).to.be.true | 108 | expect(data.import.videos.http.enabled).to.be.true |
@@ -211,6 +212,7 @@ function checkUpdatedConfig (data: CustomConfig) { | |||
211 | expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.false | 212 | expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.false |
212 | 213 | ||
213 | expect(data.videoStudio.enabled).to.be.true | 214 | expect(data.videoStudio.enabled).to.be.true |
215 | expect(data.videoStudio.remoteRunners.enabled).to.be.true | ||
214 | 216 | ||
215 | expect(data.import.videos.concurrency).to.equal(4) | 217 | expect(data.import.videos.concurrency).to.equal(4) |
216 | expect(data.import.videos.http.enabled).to.be.false | 218 | expect(data.import.videos.http.enabled).to.be.false |
@@ -374,7 +376,10 @@ const newCustomConfig: CustomConfig = { | |||
374 | } | 376 | } |
375 | }, | 377 | }, |
376 | videoStudio: { | 378 | videoStudio: { |
377 | enabled: true | 379 | enabled: true, |
380 | remoteRunners: { | ||
381 | enabled: true | ||
382 | } | ||
378 | }, | 383 | }, |
379 | import: { | 384 | import: { |
380 | videos: { | 385 | videos: { |
diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts index 30f72e6e9..2f64ef6bd 100644 --- a/server/tests/api/transcoding/video-studio.ts +++ b/server/tests/api/transcoding/video-studio.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { expect } from 'chai' | 1 | import { expect } from 'chai' |
2 | import { checkPersistentTmpIsEmpty, expectStartWith } from '@server/tests/shared' | 2 | import { checkPersistentTmpIsEmpty, checkVideoDuration, expectStartWith } from '@server/tests/shared' |
3 | import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' | 3 | import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' |
4 | import { VideoStudioTask } from '@shared/models' | 4 | import { VideoStudioTask } from '@shared/models' |
5 | import { | 5 | import { |
@@ -18,20 +18,6 @@ describe('Test video studio', function () { | |||
18 | let servers: PeerTubeServer[] = [] | 18 | let servers: PeerTubeServer[] = [] |
19 | let videoUUID: string | 19 | let videoUUID: string |
20 | 20 | ||
21 | async function checkDuration (server: PeerTubeServer, duration: number) { | ||
22 | const video = await server.videos.get({ id: videoUUID }) | ||
23 | |||
24 | expect(video.duration).to.be.approximately(duration, 1) | ||
25 | |||
26 | for (const file of video.files) { | ||
27 | const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) | ||
28 | |||
29 | for (const stream of metadata.streams) { | ||
30 | expect(Math.round(stream.duration)).to.be.approximately(duration, 1) | ||
31 | } | ||
32 | } | ||
33 | } | ||
34 | |||
35 | async function renewVideo (fixture = 'video_short.webm') { | 21 | async function renewVideo (fixture = 'video_short.webm') { |
36 | const video = await servers[0].videos.quickUpload({ name: 'video', fixture }) | 22 | const video = await servers[0].videos.quickUpload({ name: 'video', fixture }) |
37 | videoUUID = video.uuid | 23 | videoUUID = video.uuid |
@@ -79,7 +65,7 @@ describe('Test video studio', function () { | |||
79 | ]) | 65 | ]) |
80 | 66 | ||
81 | for (const server of servers) { | 67 | for (const server of servers) { |
82 | await checkDuration(server, 3) | 68 | await checkVideoDuration(server, videoUUID, 3) |
83 | 69 | ||
84 | const video = await server.videos.get({ id: videoUUID }) | 70 | const video = await server.videos.get({ id: videoUUID }) |
85 | expect(new Date(video.publishedAt)).to.be.below(beforeTasks) | 71 | expect(new Date(video.publishedAt)).to.be.below(beforeTasks) |
@@ -100,7 +86,7 @@ describe('Test video studio', function () { | |||
100 | ]) | 86 | ]) |
101 | 87 | ||
102 | for (const server of servers) { | 88 | for (const server of servers) { |
103 | await checkDuration(server, 2) | 89 | await checkVideoDuration(server, videoUUID, 2) |
104 | } | 90 | } |
105 | }) | 91 | }) |
106 | 92 | ||
@@ -119,7 +105,7 @@ describe('Test video studio', function () { | |||
119 | ]) | 105 | ]) |
120 | 106 | ||
121 | for (const server of servers) { | 107 | for (const server of servers) { |
122 | await checkDuration(server, 4) | 108 | await checkVideoDuration(server, videoUUID, 4) |
123 | } | 109 | } |
124 | }) | 110 | }) |
125 | }) | 111 | }) |
@@ -140,7 +126,7 @@ describe('Test video studio', function () { | |||
140 | ]) | 126 | ]) |
141 | 127 | ||
142 | for (const server of servers) { | 128 | for (const server of servers) { |
143 | await checkDuration(server, 10) | 129 | await checkVideoDuration(server, videoUUID, 10) |
144 | } | 130 | } |
145 | }) | 131 | }) |
146 | 132 | ||
@@ -158,7 +144,7 @@ describe('Test video studio', function () { | |||
158 | ]) | 144 | ]) |
159 | 145 | ||
160 | for (const server of servers) { | 146 | for (const server of servers) { |
161 | await checkDuration(server, 7) | 147 | await checkVideoDuration(server, videoUUID, 7) |
162 | } | 148 | } |
163 | }) | 149 | }) |
164 | 150 | ||
@@ -183,7 +169,7 @@ describe('Test video studio', function () { | |||
183 | ]) | 169 | ]) |
184 | 170 | ||
185 | for (const server of servers) { | 171 | for (const server of servers) { |
186 | await checkDuration(server, 12) | 172 | await checkVideoDuration(server, videoUUID, 12) |
187 | } | 173 | } |
188 | }) | 174 | }) |
189 | 175 | ||
@@ -201,7 +187,7 @@ describe('Test video studio', function () { | |||
201 | ]) | 187 | ]) |
202 | 188 | ||
203 | for (const server of servers) { | 189 | for (const server of servers) { |
204 | await checkDuration(server, 7) | 190 | await checkVideoDuration(server, videoUUID, 7) |
205 | } | 191 | } |
206 | }) | 192 | }) |
207 | 193 | ||
@@ -219,7 +205,7 @@ describe('Test video studio', function () { | |||
219 | ]) | 205 | ]) |
220 | 206 | ||
221 | for (const server of servers) { | 207 | for (const server of servers) { |
222 | await checkDuration(server, 10) | 208 | await checkVideoDuration(server, videoUUID, 10) |
223 | } | 209 | } |
224 | }) | 210 | }) |
225 | 211 | ||
@@ -237,7 +223,7 @@ describe('Test video studio', function () { | |||
237 | ]) | 223 | ]) |
238 | 224 | ||
239 | for (const server of servers) { | 225 | for (const server of servers) { |
240 | await checkDuration(server, 10) | 226 | await checkVideoDuration(server, videoUUID, 10) |
241 | } | 227 | } |
242 | }) | 228 | }) |
243 | }) | 229 | }) |
@@ -279,7 +265,7 @@ describe('Test video studio', function () { | |||
279 | await createTasks(VideoStudioCommand.getComplexTask()) | 265 | await createTasks(VideoStudioCommand.getComplexTask()) |
280 | 266 | ||
281 | for (const server of servers) { | 267 | for (const server of servers) { |
282 | await checkDuration(server, 9) | 268 | await checkVideoDuration(server, videoUUID, 9) |
283 | } | 269 | } |
284 | }) | 270 | }) |
285 | }) | 271 | }) |
@@ -309,7 +295,7 @@ describe('Test video studio', function () { | |||
309 | const video = await server.videos.get({ id: videoUUID }) | 295 | const video = await server.videos.get({ id: videoUUID }) |
310 | expect(video.files).to.have.lengthOf(0) | 296 | expect(video.files).to.have.lengthOf(0) |
311 | 297 | ||
312 | await checkDuration(server, 9) | 298 | await checkVideoDuration(server, videoUUID, 9) |
313 | } | 299 | } |
314 | }) | 300 | }) |
315 | }) | 301 | }) |
@@ -351,7 +337,7 @@ describe('Test video studio', function () { | |||
351 | expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) | 337 | expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) |
352 | } | 338 | } |
353 | 339 | ||
354 | await checkDuration(server, 9) | 340 | await checkVideoDuration(server, videoUUID, 9) |
355 | } | 341 | } |
356 | }) | 342 | }) |
357 | }) | 343 | }) |
@@ -370,7 +356,7 @@ describe('Test video studio', function () { | |||
370 | await waitJobs(servers) | 356 | await waitJobs(servers) |
371 | 357 | ||
372 | for (const server of servers) { | 358 | for (const server of servers) { |
373 | await checkDuration(server, 9) | 359 | await checkVideoDuration(server, videoUUID, 9) |
374 | } | 360 | } |
375 | }) | 361 | }) |
376 | 362 | ||
diff --git a/server/tests/peertube-runner/index.ts b/server/tests/peertube-runner/index.ts index 6258d6eb2..470316417 100644 --- a/server/tests/peertube-runner/index.ts +++ b/server/tests/peertube-runner/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | export * from './client-cli' | 1 | export * from './client-cli' |
2 | export * from './live-transcoding' | 2 | export * from './live-transcoding' |
3 | export * from './studio-transcoding' | ||
3 | export * from './vod-transcoding' | 4 | export * from './vod-transcoding' |
diff --git a/server/tests/peertube-runner/live-transcoding.ts b/server/tests/peertube-runner/live-transcoding.ts index f58e920ba..1e94eabcd 100644 --- a/server/tests/peertube-runner/live-transcoding.ts +++ b/server/tests/peertube-runner/live-transcoding.ts | |||
@@ -1,6 +1,12 @@ | |||
1 | import { expect } from 'chai' | 1 | import { expect } from 'chai' |
2 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 2 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
3 | import { expectStartWith, PeerTubeRunnerProcess, SQLCommand, testLiveVideoResolutions } from '@server/tests/shared' | 3 | import { |
4 | checkPeerTubeRunnerCacheIsEmpty, | ||
5 | expectStartWith, | ||
6 | PeerTubeRunnerProcess, | ||
7 | SQLCommand, | ||
8 | testLiveVideoResolutions | ||
9 | } from '@server/tests/shared' | ||
4 | import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils' | 10 | import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils' |
5 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | 11 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' |
6 | import { | 12 | import { |
@@ -169,6 +175,13 @@ describe('Test Live transcoding in peertube-runner program', function () { | |||
169 | runSuite({ objectStorage: true }) | 175 | runSuite({ objectStorage: true }) |
170 | }) | 176 | }) |
171 | 177 | ||
178 | describe('Check cleanup', function () { | ||
179 | |||
180 | it('Should have an empty cache directory', async function () { | ||
181 | await checkPeerTubeRunnerCacheIsEmpty() | ||
182 | }) | ||
183 | }) | ||
184 | |||
172 | after(async function () { | 185 | after(async function () { |
173 | await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) | 186 | await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) |
174 | peertubeRunner.kill() | 187 | peertubeRunner.kill() |
diff --git a/server/tests/peertube-runner/studio-transcoding.ts b/server/tests/peertube-runner/studio-transcoding.ts new file mode 100644 index 000000000..cca905e2f --- /dev/null +++ b/server/tests/peertube-runner/studio-transcoding.ts | |||
@@ -0,0 +1,116 @@ | |||
1 | |||
2 | import { expect } from 'chai' | ||
3 | import { checkPeerTubeRunnerCacheIsEmpty, checkVideoDuration, expectStartWith, PeerTubeRunnerProcess } from '@server/tests/shared' | ||
4 | import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' | ||
5 | import { | ||
6 | cleanupTests, | ||
7 | createMultipleServers, | ||
8 | doubleFollow, | ||
9 | ObjectStorageCommand, | ||
10 | PeerTubeServer, | ||
11 | setAccessTokensToServers, | ||
12 | setDefaultVideoChannel, | ||
13 | VideoStudioCommand, | ||
14 | waitJobs | ||
15 | } from '@shared/server-commands' | ||
16 | |||
17 | describe('Test studio transcoding in peertube-runner program', function () { | ||
18 | let servers: PeerTubeServer[] = [] | ||
19 | let peertubeRunner: PeerTubeRunnerProcess | ||
20 | |||
21 | function runSuite (options: { | ||
22 | objectStorage: boolean | ||
23 | }) { | ||
24 | const { objectStorage } = options | ||
25 | |||
26 | it('Should run a complex studio transcoding', async function () { | ||
27 | this.timeout(120000) | ||
28 | |||
29 | const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4' }) | ||
30 | await waitJobs(servers) | ||
31 | |||
32 | const video = await servers[0].videos.get({ id: uuid }) | ||
33 | const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) | ||
34 | |||
35 | await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks: VideoStudioCommand.getComplexTask() }) | ||
36 | await waitJobs(servers, { runnerJobs: true }) | ||
37 | |||
38 | for (const server of servers) { | ||
39 | const video = await server.videos.get({ id: uuid }) | ||
40 | const files = getAllFiles(video) | ||
41 | |||
42 | for (const f of files) { | ||
43 | expect(oldFileUrls).to.not.include(f.fileUrl) | ||
44 | } | ||
45 | |||
46 | if (objectStorage) { | ||
47 | for (const webtorrentFile of video.files) { | ||
48 | expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) | ||
49 | } | ||
50 | |||
51 | for (const hlsFile of video.streamingPlaylists[0].files) { | ||
52 | expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) | ||
53 | } | ||
54 | } | ||
55 | |||
56 | await checkVideoDuration(server, uuid, 9) | ||
57 | } | ||
58 | }) | ||
59 | } | ||
60 | |||
61 | before(async function () { | ||
62 | this.timeout(120_000) | ||
63 | |||
64 | servers = await createMultipleServers(2) | ||
65 | |||
66 | await setAccessTokensToServers(servers) | ||
67 | await setDefaultVideoChannel(servers) | ||
68 | |||
69 | await doubleFollow(servers[0], servers[1]) | ||
70 | |||
71 | await servers[0].config.enableTranscoding(true, true) | ||
72 | await servers[0].config.enableStudio() | ||
73 | await servers[0].config.enableRemoteStudio() | ||
74 | |||
75 | const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() | ||
76 | |||
77 | peertubeRunner = new PeerTubeRunnerProcess() | ||
78 | await peertubeRunner.runServer({ hideLogs: false }) | ||
79 | await peertubeRunner.registerPeerTubeInstance({ server: servers[0], registrationToken, runnerName: 'runner' }) | ||
80 | }) | ||
81 | |||
82 | describe('With videos on local filesystem storage', function () { | ||
83 | runSuite({ objectStorage: false }) | ||
84 | }) | ||
85 | |||
86 | describe('With videos on object storage', function () { | ||
87 | if (areMockObjectStorageTestsDisabled()) return | ||
88 | |||
89 | before(async function () { | ||
90 | await ObjectStorageCommand.prepareDefaultMockBuckets() | ||
91 | |||
92 | await servers[0].kill() | ||
93 | |||
94 | await servers[0].run(ObjectStorageCommand.getDefaultMockConfig()) | ||
95 | |||
96 | // Wait for peertube runner socket reconnection | ||
97 | await wait(1500) | ||
98 | }) | ||
99 | |||
100 | runSuite({ objectStorage: true }) | ||
101 | }) | ||
102 | |||
103 | describe('Check cleanup', function () { | ||
104 | |||
105 | it('Should have an empty cache directory', async function () { | ||
106 | await checkPeerTubeRunnerCacheIsEmpty() | ||
107 | }) | ||
108 | }) | ||
109 | |||
110 | after(async function () { | ||
111 | await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) | ||
112 | peertubeRunner.kill() | ||
113 | |||
114 | await cleanupTests(servers) | ||
115 | }) | ||
116 | }) | ||
diff --git a/server/tests/peertube-runner/vod-transcoding.ts b/server/tests/peertube-runner/vod-transcoding.ts index bdf798379..3a9abba93 100644 --- a/server/tests/peertube-runner/vod-transcoding.ts +++ b/server/tests/peertube-runner/vod-transcoding.ts | |||
@@ -1,6 +1,11 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | import { expect } from 'chai' | 2 | import { expect } from 'chai' |
3 | import { completeCheckHlsPlaylist, completeWebVideoFilesCheck, PeerTubeRunnerProcess } from '@server/tests/shared' | 3 | import { |
4 | checkPeerTubeRunnerCacheIsEmpty, | ||
5 | completeCheckHlsPlaylist, | ||
6 | completeWebVideoFilesCheck, | ||
7 | PeerTubeRunnerProcess | ||
8 | } from '@server/tests/shared' | ||
4 | import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' | 9 | import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' |
5 | import { VideoPrivacy } from '@shared/models' | 10 | import { VideoPrivacy } from '@shared/models' |
6 | import { | 11 | import { |
@@ -321,6 +326,13 @@ describe('Test VOD transcoding in peertube-runner program', function () { | |||
321 | }) | 326 | }) |
322 | }) | 327 | }) |
323 | 328 | ||
329 | describe('Check cleanup', function () { | ||
330 | |||
331 | it('Should have an empty cache directory', async function () { | ||
332 | await checkPeerTubeRunnerCacheIsEmpty() | ||
333 | }) | ||
334 | }) | ||
335 | |||
324 | after(async function () { | 336 | after(async function () { |
325 | await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) | 337 | await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) |
326 | peertubeRunner.kill() | 338 | peertubeRunner.kill() |
diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index d7eb25bb5..feaef37c6 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts | |||
@@ -130,6 +130,22 @@ function checkBadSortPagination (url: string, path: string, token?: string, quer | |||
130 | }) | 130 | }) |
131 | } | 131 | } |
132 | 132 | ||
133 | // --------------------------------------------------------------------------- | ||
134 | |||
135 | async function checkVideoDuration (server: PeerTubeServer, videoUUID: string, duration: number) { | ||
136 | const video = await server.videos.get({ id: videoUUID }) | ||
137 | |||
138 | expect(video.duration).to.be.approximately(duration, 1) | ||
139 | |||
140 | for (const file of video.files) { | ||
141 | const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) | ||
142 | |||
143 | for (const stream of metadata.streams) { | ||
144 | expect(Math.round(stream.duration)).to.be.approximately(duration, 1) | ||
145 | } | ||
146 | } | ||
147 | } | ||
148 | |||
133 | export { | 149 | export { |
134 | dateIsValid, | 150 | dateIsValid, |
135 | testImageSize, | 151 | testImageSize, |
@@ -142,5 +158,6 @@ export { | |||
142 | checkBadStartPagination, | 158 | checkBadStartPagination, |
143 | checkBadCountPagination, | 159 | checkBadCountPagination, |
144 | checkBadSortPagination, | 160 | checkBadSortPagination, |
161 | checkVideoDuration, | ||
145 | expectLogContain | 162 | expectLogContain |
146 | } | 163 | } |
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts index a614cef7c..4f4282554 100644 --- a/server/tests/shared/directories.ts +++ b/server/tests/shared/directories.ts | |||
@@ -2,9 +2,11 @@ | |||
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { pathExists, readdir } from 'fs-extra' | 4 | import { pathExists, readdir } from 'fs-extra' |
5 | import { homedir } from 'os' | ||
6 | import { join } from 'path' | ||
5 | import { PeerTubeServer } from '@shared/server-commands' | 7 | import { PeerTubeServer } from '@shared/server-commands' |
6 | 8 | ||
7 | async function checkTmpIsEmpty (server: PeerTubeServer) { | 9 | export async function checkTmpIsEmpty (server: PeerTubeServer) { |
8 | await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) | 10 | await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) |
9 | 11 | ||
10 | if (await pathExists(server.getDirectoryPath('tmp/hls'))) { | 12 | if (await pathExists(server.getDirectoryPath('tmp/hls'))) { |
@@ -12,11 +14,11 @@ async function checkTmpIsEmpty (server: PeerTubeServer) { | |||
12 | } | 14 | } |
13 | } | 15 | } |
14 | 16 | ||
15 | async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { | 17 | export async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { |
16 | await checkDirectoryIsEmpty(server, 'tmp-persistent') | 18 | await checkDirectoryIsEmpty(server, 'tmp-persistent') |
17 | } | 19 | } |
18 | 20 | ||
19 | async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { | 21 | export async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { |
20 | const directoryPath = server.getDirectoryPath(directory) | 22 | const directoryPath = server.getDirectoryPath(directory) |
21 | 23 | ||
22 | const directoryExists = await pathExists(directoryPath) | 24 | const directoryExists = await pathExists(directoryPath) |
@@ -28,8 +30,13 @@ async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, | |||
28 | expect(filtered).to.have.lengthOf(0) | 30 | expect(filtered).to.have.lengthOf(0) |
29 | } | 31 | } |
30 | 32 | ||
31 | export { | 33 | export async function checkPeerTubeRunnerCacheIsEmpty () { |
32 | checkTmpIsEmpty, | 34 | const directoryPath = join(homedir(), '.cache', 'peertube-runner-nodejs', 'test', 'transcoding') |
33 | checkPersistentTmpIsEmpty, | 35 | |
34 | checkDirectoryIsEmpty | 36 | const directoryExists = await pathExists(directoryPath) |
37 | expect(directoryExists).to.be.true | ||
38 | |||
39 | const files = await readdir(directoryPath) | ||
40 | |||
41 | expect(files).to.have.lengthOf(0) | ||
35 | } | 42 | } |
diff --git a/shared/models/runners/runner-job-payload.model.ts b/shared/models/runners/runner-job-payload.model.ts index 8f0c17135..9f0db0dc4 100644 --- a/shared/models/runners/runner-job-payload.model.ts +++ b/shared/models/runners/runner-job-payload.model.ts | |||
@@ -1,3 +1,5 @@ | |||
1 | import { VideoStudioTaskPayload } from '../server' | ||
2 | |||
1 | export type RunnerJobVODPayload = | 3 | export type RunnerJobVODPayload = |
2 | RunnerJobVODWebVideoTranscodingPayload | | 4 | RunnerJobVODWebVideoTranscodingPayload | |
3 | RunnerJobVODHLSTranscodingPayload | | 5 | RunnerJobVODHLSTranscodingPayload | |
@@ -5,7 +7,8 @@ export type RunnerJobVODPayload = | |||
5 | 7 | ||
6 | export type RunnerJobPayload = | 8 | export type RunnerJobPayload = |
7 | RunnerJobVODPayload | | 9 | RunnerJobVODPayload | |
8 | RunnerJobLiveRTMPHLSTranscodingPayload | 10 | RunnerJobLiveRTMPHLSTranscodingPayload | |
11 | RunnerJobVideoEditionTranscodingPayload | ||
9 | 12 | ||
10 | // --------------------------------------------------------------------------- | 13 | // --------------------------------------------------------------------------- |
11 | 14 | ||
@@ -43,6 +46,14 @@ export interface RunnerJobVODAudioMergeTranscodingPayload { | |||
43 | } | 46 | } |
44 | } | 47 | } |
45 | 48 | ||
49 | export interface RunnerJobVideoEditionTranscodingPayload { | ||
50 | input: { | ||
51 | videoFileUrl: string | ||
52 | } | ||
53 | |||
54 | tasks: VideoStudioTaskPayload[] | ||
55 | } | ||
56 | |||
46 | // --------------------------------------------------------------------------- | 57 | // --------------------------------------------------------------------------- |
47 | 58 | ||
48 | export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload { | 59 | export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload { |
diff --git a/shared/models/runners/runner-job-private-payload.model.ts b/shared/models/runners/runner-job-private-payload.model.ts index c1d8d1045..c8fe0a7d8 100644 --- a/shared/models/runners/runner-job-private-payload.model.ts +++ b/shared/models/runners/runner-job-private-payload.model.ts | |||
@@ -1,3 +1,5 @@ | |||
1 | import { VideoStudioTaskPayload } from '../server' | ||
2 | |||
1 | export type RunnerJobVODPrivatePayload = | 3 | export type RunnerJobVODPrivatePayload = |
2 | RunnerJobVODWebVideoTranscodingPrivatePayload | | 4 | RunnerJobVODWebVideoTranscodingPrivatePayload | |
3 | RunnerJobVODAudioMergeTranscodingPrivatePayload | | 5 | RunnerJobVODAudioMergeTranscodingPrivatePayload | |
@@ -5,7 +7,8 @@ export type RunnerJobVODPrivatePayload = | |||
5 | 7 | ||
6 | export type RunnerJobPrivatePayload = | 8 | export type RunnerJobPrivatePayload = |
7 | RunnerJobVODPrivatePayload | | 9 | RunnerJobVODPrivatePayload | |
8 | RunnerJobLiveRTMPHLSTranscodingPrivatePayload | 10 | RunnerJobLiveRTMPHLSTranscodingPrivatePayload | |
11 | RunnerJobVideoEditionTranscodingPrivatePayload | ||
9 | 12 | ||
10 | // --------------------------------------------------------------------------- | 13 | // --------------------------------------------------------------------------- |
11 | 14 | ||
@@ -32,3 +35,10 @@ export interface RunnerJobLiveRTMPHLSTranscodingPrivatePayload { | |||
32 | masterPlaylistName: string | 35 | masterPlaylistName: string |
33 | outputDirectory: string | 36 | outputDirectory: string |
34 | } | 37 | } |
38 | |||
39 | // --------------------------------------------------------------------------- | ||
40 | |||
41 | export interface RunnerJobVideoEditionTranscodingPrivatePayload { | ||
42 | videoUUID: string | ||
43 | originalTasks: VideoStudioTaskPayload[] | ||
44 | } | ||
diff --git a/shared/models/runners/runner-job-success-body.model.ts b/shared/models/runners/runner-job-success-body.model.ts index 223b7552d..17e921f69 100644 --- a/shared/models/runners/runner-job-success-body.model.ts +++ b/shared/models/runners/runner-job-success-body.model.ts | |||
@@ -11,7 +11,8 @@ export type RunnerJobSuccessPayload = | |||
11 | VODWebVideoTranscodingSuccess | | 11 | VODWebVideoTranscodingSuccess | |
12 | VODHLSTranscodingSuccess | | 12 | VODHLSTranscodingSuccess | |
13 | VODAudioMergeTranscodingSuccess | | 13 | VODAudioMergeTranscodingSuccess | |
14 | LiveRTMPHLSTranscodingSuccess | 14 | LiveRTMPHLSTranscodingSuccess | |
15 | VideoEditionTranscodingSuccess | ||
15 | 16 | ||
16 | export interface VODWebVideoTranscodingSuccess { | 17 | export interface VODWebVideoTranscodingSuccess { |
17 | videoFile: Blob | string | 18 | videoFile: Blob | string |
@@ -30,6 +31,10 @@ export interface LiveRTMPHLSTranscodingSuccess { | |||
30 | 31 | ||
31 | } | 32 | } |
32 | 33 | ||
34 | export interface VideoEditionTranscodingSuccess { | ||
35 | videoFile: Blob | string | ||
36 | } | ||
37 | |||
33 | export function isWebVideoOrAudioMergeTranscodingPayloadSuccess ( | 38 | export function isWebVideoOrAudioMergeTranscodingPayloadSuccess ( |
34 | payload: RunnerJobSuccessPayload | 39 | payload: RunnerJobSuccessPayload |
35 | ): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess { | 40 | ): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess { |
diff --git a/shared/models/runners/runner-job-type.type.ts b/shared/models/runners/runner-job-type.type.ts index 36d3b9b25..3b997cb6e 100644 --- a/shared/models/runners/runner-job-type.type.ts +++ b/shared/models/runners/runner-job-type.type.ts | |||
@@ -2,4 +2,5 @@ export type RunnerJobType = | |||
2 | 'vod-web-video-transcoding' | | 2 | 'vod-web-video-transcoding' | |
3 | 'vod-hls-transcoding' | | 3 | 'vod-hls-transcoding' | |
4 | 'vod-audio-merge-transcoding' | | 4 | 'vod-audio-merge-transcoding' | |
5 | 'live-rtmp-hls-transcoding' | 5 | 'live-rtmp-hls-transcoding' | |
6 | 'video-edition-transcoding' | ||
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 5d2c10278..4202589f3 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts | |||
@@ -165,6 +165,10 @@ export interface CustomConfig { | |||
165 | 165 | ||
166 | videoStudio: { | 166 | videoStudio: { |
167 | enabled: boolean | 167 | enabled: boolean |
168 | |||
169 | remoteRunners: { | ||
170 | enabled: boolean | ||
171 | } | ||
168 | } | 172 | } |
169 | 173 | ||
170 | import: { | 174 | import: { |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 3fd5bf7f9..22ecee324 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -225,6 +225,10 @@ export type VideoStudioTaskWatermarkPayload = { | |||
225 | 225 | ||
226 | options: { | 226 | options: { |
227 | file: string | 227 | file: string |
228 | |||
229 | watermarkSizeRatio: number | ||
230 | horitonzalMarginRatio: number | ||
231 | verticalMarginRatio: number | ||
228 | } | 232 | } |
229 | } | 233 | } |
230 | 234 | ||
diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index 38b9d0385..024ed35bf 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { VideoPrivacy } from '../videos/video-privacy.enum' | ||
2 | import { ClientScriptJSON } from '../plugins/plugin-package-json.model' | 1 | import { ClientScriptJSON } from '../plugins/plugin-package-json.model' |
3 | import { NSFWPolicyType } from '../videos/nsfw-policy.type' | 2 | import { NSFWPolicyType } from '../videos/nsfw-policy.type' |
3 | import { VideoPrivacy } from '../videos/video-privacy.enum' | ||
4 | import { BroadcastMessageLevel } from './broadcast-message-level.type' | 4 | import { BroadcastMessageLevel } from './broadcast-message-level.type' |
5 | 5 | ||
6 | export interface ServerConfigPlugin { | 6 | export interface ServerConfigPlugin { |
@@ -186,6 +186,10 @@ export interface ServerConfig { | |||
186 | 186 | ||
187 | videoStudio: { | 187 | videoStudio: { |
188 | enabled: boolean | 188 | enabled: boolean |
189 | |||
190 | remoteRunners: { | ||
191 | enabled: boolean | ||
192 | } | ||
189 | } | 193 | } |
190 | 194 | ||
191 | import: { | 195 | import: { |
diff --git a/shared/models/videos/studio/video-studio-create-edit.model.ts b/shared/models/videos/studio/video-studio-create-edit.model.ts index 001d65c90..5e8296dc9 100644 --- a/shared/models/videos/studio/video-studio-create-edit.model.ts +++ b/shared/models/videos/studio/video-studio-create-edit.model.ts | |||
@@ -40,3 +40,21 @@ export interface VideoStudioTaskWatermark { | |||
40 | file: Blob | string | 40 | file: Blob | string |
41 | } | 41 | } |
42 | } | 42 | } |
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | export function isVideoStudioTaskIntro (v: VideoStudioTask): v is VideoStudioTaskIntro { | ||
47 | return v.name === 'add-intro' | ||
48 | } | ||
49 | |||
50 | export function isVideoStudioTaskOutro (v: VideoStudioTask): v is VideoStudioTaskOutro { | ||
51 | return v.name === 'add-outro' | ||
52 | } | ||
53 | |||
54 | export function isVideoStudioTaskWatermark (v: VideoStudioTask): v is VideoStudioTaskWatermark { | ||
55 | return v.name === 'add-watermark' | ||
56 | } | ||
57 | |||
58 | export function hasVideoStudioTaskFile (v: VideoStudioTask): v is VideoStudioTaskIntro | VideoStudioTaskOutro | VideoStudioTaskWatermark { | ||
59 | return isVideoStudioTaskIntro(v) || isVideoStudioTaskOutro(v) || isVideoStudioTaskWatermark(v) | ||
60 | } | ||
diff --git a/shared/server-commands/runners/runner-jobs-command.ts b/shared/server-commands/runners/runner-jobs-command.ts index 3b0f84b9d..26dbef77a 100644 --- a/shared/server-commands/runners/runner-jobs-command.ts +++ b/shared/server-commands/runners/runner-jobs-command.ts | |||
@@ -200,7 +200,7 @@ export class RunnerJobsCommand extends AbstractCommand { | |||
200 | }) | 200 | }) |
201 | } | 201 | } |
202 | 202 | ||
203 | getInputFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) { | 203 | getJobFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) { |
204 | const { host, protocol, pathname } = new URL(options.url) | 204 | const { host, protocol, pathname } = new URL(options.url) |
205 | 205 | ||
206 | return this.postBodyRequest({ | 206 | return this.postBodyRequest({ |
@@ -249,8 +249,15 @@ export class RunnerJobsCommand extends AbstractCommand { | |||
249 | 249 | ||
250 | const { data } = await this.list({ count: 100 }) | 250 | const { data } = await this.list({ count: 100 }) |
251 | 251 | ||
252 | const allowedStates = new Set<RunnerJobState>([ | ||
253 | RunnerJobState.PENDING, | ||
254 | RunnerJobState.PROCESSING, | ||
255 | RunnerJobState.WAITING_FOR_PARENT_JOB | ||
256 | ]) | ||
257 | |||
252 | for (const job of data) { | 258 | for (const job of data) { |
253 | if (state && job.state.id !== state) continue | 259 | if (state && job.state.id !== state) continue |
260 | else if (allowedStates.has(job.state.id) !== true) continue | ||
254 | 261 | ||
255 | await this.cancelByAdmin({ jobUUID: job.uuid }) | 262 | await this.cancelByAdmin({ jobUUID: job.uuid }) |
256 | } | 263 | } |
diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts index 9a6e413f2..b94bd2625 100644 --- a/shared/server-commands/server/config-command.ts +++ b/shared/server-commands/server/config-command.ts | |||
@@ -195,6 +195,18 @@ export class ConfigCommand extends AbstractCommand { | |||
195 | }) | 195 | }) |
196 | } | 196 | } |
197 | 197 | ||
198 | enableRemoteStudio () { | ||
199 | return this.updateExistingSubConfig({ | ||
200 | newConfig: { | ||
201 | videoStudio: { | ||
202 | remoteRunners: { | ||
203 | enabled: true | ||
204 | } | ||
205 | } | ||
206 | } | ||
207 | }) | ||
208 | } | ||
209 | |||
198 | // --------------------------------------------------------------------------- | 210 | // --------------------------------------------------------------------------- |
199 | 211 | ||
200 | enableStudio () { | 212 | enableStudio () { |
@@ -442,7 +454,10 @@ export class ConfigCommand extends AbstractCommand { | |||
442 | } | 454 | } |
443 | }, | 455 | }, |
444 | videoStudio: { | 456 | videoStudio: { |
445 | enabled: false | 457 | enabled: false, |
458 | remoteRunners: { | ||
459 | enabled: false | ||
460 | } | ||
446 | }, | 461 | }, |
447 | import: { | 462 | import: { |
448 | videos: { | 463 | videos: { |