diff options
Diffstat (limited to 'packages/peertube-runner/server')
7 files changed, 84 insertions, 205 deletions
diff --git a/packages/peertube-runner/server/process/process.ts b/packages/peertube-runner/server/process/process.ts index ef231cb38..1caafda8c 100644 --- a/packages/peertube-runner/server/process/process.ts +++ b/packages/peertube-runner/server/process/process.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { logger } from 'packages/peertube-runner/shared/logger' | 1 | import { logger } from 'packages/peertube-runner/shared/logger' |
2 | import { | 2 | import { |
3 | RunnerJobLiveRTMPHLSTranscodingPayload, | 3 | RunnerJobLiveRTMPHLSTranscodingPayload, |
4 | RunnerJobVideoEditionTranscodingPayload, | 4 | RunnerJobStudioTranscodingPayload, |
5 | RunnerJobVODAudioMergeTranscodingPayload, | 5 | RunnerJobVODAudioMergeTranscodingPayload, |
6 | RunnerJobVODHLSTranscodingPayload, | 6 | RunnerJobVODHLSTranscodingPayload, |
7 | RunnerJobVODWebVideoTranscodingPayload | 7 | RunnerJobVODWebVideoTranscodingPayload |
@@ -23,8 +23,8 @@ export async function processJob (options: ProcessOptions) { | |||
23 | await processHLSTranscoding(options as ProcessOptions<RunnerJobVODHLSTranscodingPayload>) | 23 | await processHLSTranscoding(options as ProcessOptions<RunnerJobVODHLSTranscodingPayload>) |
24 | } else if (job.type === 'live-rtmp-hls-transcoding') { | 24 | } else if (job.type === 'live-rtmp-hls-transcoding') { |
25 | await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>).process() | 25 | await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>).process() |
26 | } else if (job.type === 'video-edition-transcoding') { | 26 | } else if (job.type === 'video-studio-transcoding') { |
27 | await processStudioTranscoding(options as ProcessOptions<RunnerJobVideoEditionTranscodingPayload>) | 27 | await processStudioTranscoding(options as ProcessOptions<RunnerJobStudioTranscodingPayload>) |
28 | } else { | 28 | } else { |
29 | logger.error(`Unknown job ${job.type} to process`) | 29 | logger.error(`Unknown job ${job.type} to process`) |
30 | return | 30 | return |
diff --git a/packages/peertube-runner/server/process/shared/common.ts b/packages/peertube-runner/server/process/shared/common.ts index 3cac98388..88f7c33f1 100644 --- a/packages/peertube-runner/server/process/shared/common.ts +++ b/packages/peertube-runner/server/process/shared/common.ts | |||
@@ -1,13 +1,12 @@ | |||
1 | import { remove } from 'fs-extra' | ||
1 | import { throttle } from 'lodash' | 2 | import { throttle } from 'lodash' |
2 | import { ConfigManager, downloadFile, logger } from 'packages/peertube-runner/shared' | 3 | import { ConfigManager, downloadFile, logger } from 'packages/peertube-runner/shared' |
3 | import { join } from 'path' | 4 | import { join } from 'path' |
4 | import { buildUUID } from '@shared/extra-utils' | 5 | import { buildUUID } from '@shared/extra-utils' |
5 | import { FFmpegEdition, FFmpegLive, FFmpegVOD } from '@shared/ffmpeg' | 6 | import { FFmpegEdition, FFmpegLive, FFmpegVOD, getDefaultAvailableEncoders, getDefaultEncodersToTry } from '@shared/ffmpeg' |
6 | import { RunnerJob, RunnerJobPayload } from '@shared/models' | 7 | import { RunnerJob, RunnerJobPayload } from '@shared/models' |
7 | import { PeerTubeServer } from '@shared/server-commands' | 8 | import { PeerTubeServer } from '@shared/server-commands' |
8 | import { getTranscodingLogger } from './transcoding-logger' | 9 | import { getTranscodingLogger } from './transcoding-logger' |
9 | import { getAvailableEncoders, getEncodersToTry } from './transcoding-profiles' | ||
10 | import { remove } from 'fs-extra' | ||
11 | 10 | ||
12 | export type JobWithToken <T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string } | 11 | export type JobWithToken <T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string } |
13 | 12 | ||
@@ -92,8 +91,8 @@ function getCommonFFmpegOptions () { | |||
92 | tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), | 91 | tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), |
93 | profile: 'default', | 92 | profile: 'default', |
94 | availableEncoders: { | 93 | availableEncoders: { |
95 | available: getAvailableEncoders(), | 94 | available: getDefaultAvailableEncoders(), |
96 | encodersToTry: getEncodersToTry() | 95 | encodersToTry: getDefaultEncodersToTry() |
97 | }, | 96 | }, |
98 | logger: getTranscodingLogger() | 97 | logger: getTranscodingLogger() |
99 | } | 98 | } |
diff --git a/packages/peertube-runner/server/process/shared/index.ts b/packages/peertube-runner/server/process/shared/index.ts index 8e09a7869..556c51365 100644 --- a/packages/peertube-runner/server/process/shared/index.ts +++ b/packages/peertube-runner/server/process/shared/index.ts | |||
@@ -1,4 +1,3 @@ | |||
1 | export * from './common' | 1 | export * from './common' |
2 | export * from './process-vod' | 2 | export * from './process-vod' |
3 | export * from './transcoding-logger' | 3 | export * from './transcoding-logger' |
4 | export * from './transcoding-profiles' | ||
diff --git a/packages/peertube-runner/server/process/shared/process-studio.ts b/packages/peertube-runner/server/process/shared/process-studio.ts index f8262096e..9c745d031 100644 --- a/packages/peertube-runner/server/process/shared/process-studio.ts +++ b/packages/peertube-runner/server/process/shared/process-studio.ts | |||
@@ -1,11 +1,11 @@ | |||
1 | import { remove } from 'fs-extra' | 1 | import { remove } from 'fs-extra' |
2 | import { pick } from 'lodash' | 2 | import { pick } from 'lodash' |
3 | import { logger } from 'packages/peertube-runner/shared' | 3 | import { logger } from 'packages/peertube-runner/shared' |
4 | import { extname, join } from 'path' | 4 | import { join } from 'path' |
5 | import { buildUUID } from '@shared/extra-utils' | 5 | import { buildUUID } from '@shared/extra-utils' |
6 | import { | 6 | import { |
7 | RunnerJobVideoEditionTranscodingPayload, | 7 | RunnerJobStudioTranscodingPayload, |
8 | VideoEditionTranscodingSuccess, | 8 | VideoStudioTranscodingSuccess, |
9 | VideoStudioTask, | 9 | VideoStudioTask, |
10 | VideoStudioTaskCutPayload, | 10 | VideoStudioTaskCutPayload, |
11 | VideoStudioTaskIntroPayload, | 11 | VideoStudioTaskIntroPayload, |
@@ -16,7 +16,7 @@ import { | |||
16 | import { ConfigManager } from '../../../shared/config-manager' | 16 | import { ConfigManager } from '../../../shared/config-manager' |
17 | import { buildFFmpegEdition, downloadInputFile, JobWithToken, ProcessOptions } from './common' | 17 | import { buildFFmpegEdition, downloadInputFile, JobWithToken, ProcessOptions } from './common' |
18 | 18 | ||
19 | export async function processStudioTranscoding (options: ProcessOptions<RunnerJobVideoEditionTranscodingPayload>) { | 19 | export async function processStudioTranscoding (options: ProcessOptions<RunnerJobStudioTranscodingPayload>) { |
20 | const { server, job, runnerToken } = options | 20 | const { server, job, runnerToken } = options |
21 | const payload = job.payload | 21 | const payload = job.payload |
22 | 22 | ||
@@ -43,7 +43,7 @@ export async function processStudioTranscoding (options: ProcessOptions<RunnerJo | |||
43 | tmpInputFilePath = outputPath | 43 | tmpInputFilePath = outputPath |
44 | } | 44 | } |
45 | 45 | ||
46 | const successBody: VideoEditionTranscodingSuccess = { | 46 | const successBody: VideoStudioTranscodingSuccess = { |
47 | videoFile: outputPath | 47 | videoFile: outputPath |
48 | } | 48 | } |
49 | 49 | ||
@@ -94,14 +94,18 @@ async function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTa | |||
94 | 94 | ||
95 | const introOutroPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) | 95 | const introOutroPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) |
96 | 96 | ||
97 | return buildFFmpegEdition().addIntroOutro({ | 97 | try { |
98 | ...pick(options, [ 'inputPath', 'outputPath' ]), | 98 | await buildFFmpegEdition().addIntroOutro({ |
99 | ...pick(options, [ 'inputPath', 'outputPath' ]), | ||
99 | 100 | ||
100 | introOutroPath, | 101 | introOutroPath, |
101 | type: task.name === 'add-intro' | 102 | type: task.name === 'add-intro' |
102 | ? 'intro' | 103 | ? 'intro' |
103 | : 'outro' | 104 | : 'outro' |
104 | }) | 105 | }) |
106 | } finally { | ||
107 | await remove(introOutroPath) | ||
108 | } | ||
105 | } | 109 | } |
106 | 110 | ||
107 | function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) { | 111 | function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) { |
@@ -124,15 +128,19 @@ async function processAddWatermark (options: TaskProcessorOptions<VideoStudioTas | |||
124 | 128 | ||
125 | const watermarkPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) | 129 | const watermarkPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) |
126 | 130 | ||
127 | return buildFFmpegEdition().addWatermark({ | 131 | try { |
128 | ...pick(options, [ 'inputPath', 'outputPath' ]), | 132 | await buildFFmpegEdition().addWatermark({ |
133 | ...pick(options, [ 'inputPath', 'outputPath' ]), | ||
129 | 134 | ||
130 | watermarkPath, | 135 | watermarkPath, |
131 | 136 | ||
132 | videoFilters: { | 137 | videoFilters: { |
133 | watermarkSizeRatio: task.options.watermarkSizeRatio, | 138 | watermarkSizeRatio: task.options.watermarkSizeRatio, |
134 | horitonzalMarginRatio: task.options.horitonzalMarginRatio, | 139 | horitonzalMarginRatio: task.options.horitonzalMarginRatio, |
135 | verticalMarginRatio: task.options.verticalMarginRatio | 140 | verticalMarginRatio: task.options.verticalMarginRatio |
136 | } | 141 | } |
137 | }) | 142 | }) |
143 | } finally { | ||
144 | await remove(watermarkPath) | ||
145 | } | ||
138 | } | 146 | } |
diff --git a/packages/peertube-runner/server/process/shared/process-vod.ts b/packages/peertube-runner/server/process/shared/process-vod.ts index d84ece3cb..22489afd5 100644 --- a/packages/peertube-runner/server/process/shared/process-vod.ts +++ b/packages/peertube-runner/server/process/shared/process-vod.ts | |||
@@ -22,31 +22,34 @@ export async function processWebVideoTranscoding (options: ProcessOptions<Runner | |||
22 | 22 | ||
23 | const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`) | 23 | const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`) |
24 | 24 | ||
25 | await ffmpegVod.transcode({ | 25 | try { |
26 | type: 'video', | 26 | await ffmpegVod.transcode({ |
27 | 27 | type: 'video', | |
28 | inputPath, | ||
29 | 28 | ||
30 | outputPath, | 29 | inputPath, |
31 | 30 | ||
32 | inputFileMutexReleaser: () => {}, | 31 | outputPath, |
33 | 32 | ||
34 | resolution: payload.output.resolution, | 33 | inputFileMutexReleaser: () => {}, |
35 | fps: payload.output.fps | ||
36 | }) | ||
37 | 34 | ||
38 | const successBody: VODWebVideoTranscodingSuccess = { | 35 | resolution: payload.output.resolution, |
39 | videoFile: outputPath | 36 | fps: payload.output.fps |
40 | } | 37 | }) |
41 | 38 | ||
42 | await server.runnerJobs.success({ | 39 | const successBody: VODWebVideoTranscodingSuccess = { |
43 | jobToken: job.jobToken, | 40 | videoFile: outputPath |
44 | jobUUID: job.uuid, | 41 | } |
45 | runnerToken, | ||
46 | payload: successBody | ||
47 | }) | ||
48 | 42 | ||
49 | await remove(outputPath) | 43 | await server.runnerJobs.success({ |
44 | jobToken: job.jobToken, | ||
45 | jobUUID: job.uuid, | ||
46 | runnerToken, | ||
47 | payload: successBody | ||
48 | }) | ||
49 | } finally { | ||
50 | await remove(inputPath) | ||
51 | await remove(outputPath) | ||
52 | } | ||
50 | } | 53 | } |
51 | 54 | ||
52 | export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVODHLSTranscodingPayload>) { | 55 | export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVODHLSTranscodingPayload>) { |
@@ -105,30 +108,34 @@ export async function processAudioMergeTranscoding (options: ProcessOptions<Runn | |||
105 | 108 | ||
106 | const ffmpegVod = buildFFmpegVOD({ job, server, runnerToken }) | 109 | const ffmpegVod = buildFFmpegVOD({ job, server, runnerToken }) |
107 | 110 | ||
108 | await ffmpegVod.transcode({ | 111 | try { |
109 | type: 'merge-audio', | 112 | await ffmpegVod.transcode({ |
110 | 113 | type: 'merge-audio', | |
111 | audioPath, | ||
112 | inputPath, | ||
113 | 114 | ||
114 | outputPath, | 115 | audioPath, |
116 | inputPath, | ||
115 | 117 | ||
116 | inputFileMutexReleaser: () => {}, | 118 | outputPath, |
117 | 119 | ||
118 | resolution: payload.output.resolution, | 120 | inputFileMutexReleaser: () => {}, |
119 | fps: payload.output.fps | ||
120 | }) | ||
121 | 121 | ||
122 | const successBody: VODAudioMergeTranscodingSuccess = { | 122 | resolution: payload.output.resolution, |
123 | videoFile: outputPath | 123 | fps: payload.output.fps |
124 | } | 124 | }) |
125 | 125 | ||
126 | await server.runnerJobs.success({ | 126 | const successBody: VODAudioMergeTranscodingSuccess = { |
127 | jobToken: job.jobToken, | 127 | videoFile: outputPath |
128 | jobUUID: job.uuid, | 128 | } |
129 | runnerToken, | ||
130 | payload: successBody | ||
131 | }) | ||
132 | 129 | ||
133 | await remove(outputPath) | 130 | await server.runnerJobs.success({ |
131 | jobToken: job.jobToken, | ||
132 | jobUUID: job.uuid, | ||
133 | runnerToken, | ||
134 | payload: successBody | ||
135 | }) | ||
136 | } finally { | ||
137 | await remove(audioPath) | ||
138 | await remove(inputPath) | ||
139 | await remove(outputPath) | ||
140 | } | ||
134 | } | 141 | } |
diff --git a/packages/peertube-runner/server/process/shared/transcoding-profiles.ts b/packages/peertube-runner/server/process/shared/transcoding-profiles.ts deleted file mode 100644 index 492d17d6a..000000000 --- a/packages/peertube-runner/server/process/shared/transcoding-profiles.ts +++ /dev/null | |||
@@ -1,134 +0,0 @@ | |||
1 | import { getAverageBitrate, getMinLimitBitrate } from '@shared/core-utils' | ||
2 | import { buildStreamSuffix, ffprobePromise, getAudioStream, getMaxAudioBitrate } from '@shared/ffmpeg' | ||
3 | import { EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '@shared/models' | ||
4 | |||
5 | const defaultX264VODOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOptionsBuilderParams) => { | ||
6 | const { fps, inputRatio, inputBitrate, resolution } = options | ||
7 | |||
8 | const targetBitrate = getTargetBitrate({ inputBitrate, ratio: inputRatio, fps, resolution }) | ||
9 | |||
10 | return { | ||
11 | outputOptions: [ | ||
12 | ...getCommonOutputOptions(targetBitrate), | ||
13 | |||
14 | `-r ${fps}` | ||
15 | ] | ||
16 | } | ||
17 | } | ||
18 | |||
19 | const defaultX264LiveOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOptionsBuilderParams) => { | ||
20 | const { streamNum, fps, inputBitrate, inputRatio, resolution } = options | ||
21 | |||
22 | const targetBitrate = getTargetBitrate({ inputBitrate, ratio: inputRatio, fps, resolution }) | ||
23 | |||
24 | return { | ||
25 | outputOptions: [ | ||
26 | ...getCommonOutputOptions(targetBitrate, streamNum), | ||
27 | |||
28 | `${buildStreamSuffix('-r:v', streamNum)} ${fps}`, | ||
29 | `${buildStreamSuffix('-b:v', streamNum)} ${targetBitrate}` | ||
30 | ] | ||
31 | } | ||
32 | } | ||
33 | |||
34 | const defaultAACOptionsBuilder: EncoderOptionsBuilder = async ({ input, streamNum, canCopyAudio }) => { | ||
35 | const probe = await ffprobePromise(input) | ||
36 | |||
37 | const parsedAudio = await getAudioStream(input, probe) | ||
38 | |||
39 | // We try to reduce the ceiling bitrate by making rough matches of bitrates | ||
40 | // Of course this is far from perfect, but it might save some space in the end | ||
41 | |||
42 | const audioCodecName = parsedAudio.audioStream['codec_name'] | ||
43 | |||
44 | const bitrate = getMaxAudioBitrate(audioCodecName, parsedAudio.bitrate) | ||
45 | |||
46 | // Force stereo as it causes some issues with HLS playback in Chrome | ||
47 | const base = [ '-channel_layout', 'stereo' ] | ||
48 | |||
49 | if (bitrate !== -1) { | ||
50 | return { outputOptions: base.concat([ buildStreamSuffix('-b:a', streamNum), bitrate + 'k' ]) } | ||
51 | } | ||
52 | |||
53 | return { outputOptions: base } | ||
54 | } | ||
55 | |||
56 | const defaultLibFDKAACVODOptionsBuilder: EncoderOptionsBuilder = ({ streamNum }) => { | ||
57 | return { outputOptions: [ buildStreamSuffix('-q:a', streamNum), '5' ] } | ||
58 | } | ||
59 | |||
60 | export function getAvailableEncoders () { | ||
61 | return { | ||
62 | vod: { | ||
63 | libx264: { | ||
64 | default: defaultX264VODOptionsBuilder | ||
65 | }, | ||
66 | aac: { | ||
67 | default: defaultAACOptionsBuilder | ||
68 | }, | ||
69 | libfdk_aac: { | ||
70 | default: defaultLibFDKAACVODOptionsBuilder | ||
71 | } | ||
72 | }, | ||
73 | live: { | ||
74 | libx264: { | ||
75 | default: defaultX264LiveOptionsBuilder | ||
76 | }, | ||
77 | aac: { | ||
78 | default: defaultAACOptionsBuilder | ||
79 | } | ||
80 | } | ||
81 | } | ||
82 | } | ||
83 | |||
84 | export function getEncodersToTry () { | ||
85 | return { | ||
86 | vod: { | ||
87 | video: [ 'libx264' ], | ||
88 | audio: [ 'libfdk_aac', 'aac' ] | ||
89 | }, | ||
90 | |||
91 | live: { | ||
92 | video: [ 'libx264' ], | ||
93 | audio: [ 'libfdk_aac', 'aac' ] | ||
94 | } | ||
95 | } | ||
96 | } | ||
97 | |||
98 | // --------------------------------------------------------------------------- | ||
99 | |||
100 | function getTargetBitrate (options: { | ||
101 | inputBitrate: number | ||
102 | resolution: VideoResolution | ||
103 | ratio: number | ||
104 | fps: number | ||
105 | }) { | ||
106 | const { inputBitrate, resolution, ratio, fps } = options | ||
107 | |||
108 | const capped = capBitrate(inputBitrate, getAverageBitrate({ resolution, fps, ratio })) | ||
109 | const limit = getMinLimitBitrate({ resolution, fps, ratio }) | ||
110 | |||
111 | return Math.max(limit, capped) | ||
112 | } | ||
113 | |||
114 | function capBitrate (inputBitrate: number, targetBitrate: number) { | ||
115 | if (!inputBitrate) return targetBitrate | ||
116 | |||
117 | // Add 30% margin to input bitrate | ||
118 | const inputBitrateWithMargin = inputBitrate + (inputBitrate * 0.3) | ||
119 | |||
120 | return Math.min(targetBitrate, inputBitrateWithMargin) | ||
121 | } | ||
122 | |||
123 | function getCommonOutputOptions (targetBitrate: number, streamNum?: number) { | ||
124 | return [ | ||
125 | `-preset veryfast`, | ||
126 | `${buildStreamSuffix('-maxrate:v', streamNum)} ${targetBitrate}`, | ||
127 | `${buildStreamSuffix('-bufsize:v', streamNum)} ${targetBitrate * 2}`, | ||
128 | |||
129 | // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it | ||
130 | `-b_strategy 1`, | ||
131 | // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16 | ||
132 | `-bf 16` | ||
133 | ] | ||
134 | } | ||
diff --git a/packages/peertube-runner/server/shared/supported-job.ts b/packages/peertube-runner/server/shared/supported-job.ts index 87d5a39cc..1137d8206 100644 --- a/packages/peertube-runner/server/shared/supported-job.ts +++ b/packages/peertube-runner/server/shared/supported-job.ts | |||
@@ -2,7 +2,7 @@ import { | |||
2 | RunnerJobLiveRTMPHLSTranscodingPayload, | 2 | RunnerJobLiveRTMPHLSTranscodingPayload, |
3 | RunnerJobPayload, | 3 | RunnerJobPayload, |
4 | RunnerJobType, | 4 | RunnerJobType, |
5 | RunnerJobVideoEditionTranscodingPayload, | 5 | RunnerJobStudioTranscodingPayload, |
6 | RunnerJobVODAudioMergeTranscodingPayload, | 6 | RunnerJobVODAudioMergeTranscodingPayload, |
7 | RunnerJobVODHLSTranscodingPayload, | 7 | RunnerJobVODHLSTranscodingPayload, |
8 | RunnerJobVODWebVideoTranscodingPayload, | 8 | RunnerJobVODWebVideoTranscodingPayload, |
@@ -22,7 +22,7 @@ const supportedMatrix = { | |||
22 | 'live-rtmp-hls-transcoding': (_payload: RunnerJobLiveRTMPHLSTranscodingPayload) => { | 22 | 'live-rtmp-hls-transcoding': (_payload: RunnerJobLiveRTMPHLSTranscodingPayload) => { |
23 | return true | 23 | return true |
24 | }, | 24 | }, |
25 | 'video-edition-transcoding': (payload: RunnerJobVideoEditionTranscodingPayload) => { | 25 | 'video-studio-transcoding': (payload: RunnerJobStudioTranscodingPayload) => { |
26 | const tasks = payload?.tasks | 26 | const tasks = payload?.tasks |
27 | const supported = new Set<VideoStudioTaskPayload['name']>([ 'add-intro', 'add-outro', 'add-watermark', 'cut' ]) | 27 | const supported = new Set<VideoStudioTaskPayload['name']>([ 'add-intro', 'add-outro', 'add-watermark', 'cut' ]) |
28 | 28 | ||