aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2023-05-04 15:29:34 +0200
committerChocobozzz <chocobozzz@cpy.re>2023-05-09 08:57:34 +0200
commit5e47f6ab984a7d00782e4c7030afffa1ba480add (patch)
tree1ce586b591a8d71acbc301eba29b9a5e6490439e /server/lib
parent6a4905602636afd6650c9e6f4d0fcc2105d91100 (diff)
downloadPeerTube-5e47f6ab984a7d00782e4c7030afffa1ba480add.tar.gz
PeerTube-5e47f6ab984a7d00782e4c7030afffa1ba480add.tar.zst
PeerTube-5e47f6ab984a7d00782e4c7030afffa1ba480add.zip
Support studio transcoding in peertube runner
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/job-queue/handlers/video-studio-edition.ts79
-rw-r--r--server/lib/runners/job-handlers/abstract-job-handler.ts11
-rw-r--r--server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts10
-rw-r--r--server/lib/runners/job-handlers/index.ts3
-rw-r--r--server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts2
-rw-r--r--server/lib/runners/job-handlers/runner-job-handlers.ts4
-rw-r--r--server/lib/runners/job-handlers/video-edition-transcoding-job-handler.ts157
-rw-r--r--server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts2
-rw-r--r--server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts2
-rw-r--r--server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts2
-rw-r--r--server/lib/runners/runner-urls.ts4
-rw-r--r--server/lib/server-config-manager.ts5
-rw-r--r--server/lib/transcoding/shared/job-builders/abstract-job-builder.ts18
-rw-r--r--server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts3
-rw-r--r--server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts15
-rw-r--r--server/lib/transcoding/transcoding-priority.ts24
-rw-r--r--server/lib/video-studio.ts109
17 files changed, 326 insertions, 124 deletions
diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts
index 5e8dd4f51..df73caf72 100644
--- a/server/lib/job-queue/handlers/video-studio-edition.ts
+++ b/server/lib/job-queue/handlers/video-studio-edition.ts
@@ -1,25 +1,18 @@
1import { Job } from 'bullmq' 1import { Job } from 'bullmq'
2import { move, remove } from 'fs-extra' 2import { remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' 4import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
5import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config' 5import { CONFIG } from '@server/initializers/config'
7import { VIDEO_FILTERS } from '@server/initializers/constants'
8import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
9import { generateWebTorrentVideoFilename } from '@server/lib/paths'
10import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job'
11import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' 6import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles'
12import { isAbleToUploadVideo } from '@server/lib/user' 7import { isAbleToUploadVideo } from '@server/lib/user'
13import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
14import { VideoPathManager } from '@server/lib/video-path-manager' 8import { VideoPathManager } from '@server/lib/video-path-manager'
15import { approximateIntroOutroAdditionalSize, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' 9import { approximateIntroOutroAdditionalSize, onVideoEditionEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio'
16import { UserModel } from '@server/models/user/user' 10import { UserModel } from '@server/models/user/user'
17import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
18import { VideoFileModel } from '@server/models/video/video-file' 12import { MVideo, MVideoFullLight } from '@server/types/models'
19import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models' 13import { pick } from '@shared/core-utils'
20import { getLowercaseExtension, pick } from '@shared/core-utils' 14import { buildUUID } from '@shared/extra-utils'
21import { buildUUID, getFileSize } from '@shared/extra-utils' 15import { FFmpegEdition } from '@shared/ffmpeg'
22import { FFmpegEdition, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg'
23import { 16import {
24 VideoStudioEditionPayload, 17 VideoStudioEditionPayload,
25 VideoStudioTask, 18 VideoStudioTask,
@@ -46,7 +39,7 @@ async function processVideoStudioEdition (job: Job) {
46 if (!video) { 39 if (!video) {
47 logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) 40 logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
48 41
49 await safeCleanupStudioTMPFiles(payload) 42 await safeCleanupStudioTMPFiles(payload.tasks)
50 return undefined 43 return undefined
51 } 44 }
52 45
@@ -81,28 +74,9 @@ async function processVideoStudioEdition (job: Job) {
81 74
82 logger.info('Video edition ended for video %s.', video.uuid, lTags) 75 logger.info('Video edition ended for video %s.', video.uuid, lTags)
83 76
84 const newFile = await buildNewFile(video, editionResultPath) 77 await onVideoEditionEnded({ video, editionResultPath, tasks: payload.tasks })
85
86 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile)
87 await move(editionResultPath, outputPath)
88
89 await safeCleanupStudioTMPFiles(payload)
90
91 await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)
92 await removeAllFiles(video, newFile)
93
94 await newFile.save()
95
96 video.duration = await getVideoStreamDuration(outputPath)
97 await video.save()
98
99 await federateVideoIfNeeded(video, false, undefined)
100
101 const user = await UserModel.loadByVideoId(video.id)
102
103 await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false })
104 } catch (err) { 78 } catch (err) {
105 await safeCleanupStudioTMPFiles(payload) 79 await safeCleanupStudioTMPFiles(payload.tasks)
106 80
107 throw err 81 throw err
108 } 82 }
@@ -181,44 +155,15 @@ function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWater
181 watermarkPath: task.options.file, 155 watermarkPath: task.options.file,
182 156
183 videoFilters: { 157 videoFilters: {
184 watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO, 158 watermarkSizeRatio: task.options.watermarkSizeRatio,
185 horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO, 159 horitonzalMarginRatio: task.options.horitonzalMarginRatio,
186 verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO 160 verticalMarginRatio: task.options.verticalMarginRatio
187 } 161 }
188 }) 162 })
189} 163}
190 164
191// --------------------------------------------------------------------------- 165// ---------------------------------------------------------------------------
192 166
193async function buildNewFile (video: MVideoId, path: string) {
194 const videoFile = new VideoFileModel({
195 extname: getLowercaseExtension(path),
196 size: await getFileSize(path),
197 metadata: await buildFileMetadata(path),
198 videoStreamingPlaylistId: null,
199 videoId: video.id
200 })
201
202 const probe = await ffprobePromise(path)
203
204 videoFile.fps = await getVideoStreamFPS(path, probe)
205 videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution
206
207 videoFile.filename = generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname)
208
209 return videoFile
210}
211
212async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) {
213 await removeHLSPlaylist(video)
214
215 for (const file of video.VideoFiles) {
216 if (file.id === webTorrentFileException.id) continue
217
218 await removeWebTorrentFile(video, file.id)
219 }
220}
221
222async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) { 167async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) {
223 const user = await UserModel.loadByVideoId(video.id) 168 const user = await UserModel.loadByVideoId(video.id)
224 169
diff --git a/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/lib/runners/job-handlers/abstract-job-handler.ts
index 74b455107..76fd1c5ac 100644
--- a/server/lib/runners/job-handlers/abstract-job-handler.ts
+++ b/server/lib/runners/job-handlers/abstract-job-handler.ts
@@ -1,3 +1,4 @@
1import { throttle } from 'lodash'
1import { retryTransactionWrapper } from '@server/helpers/database-utils' 2import { retryTransactionWrapper } from '@server/helpers/database-utils'
2import { logger, loggerTagsFactory } from '@server/helpers/logger' 3import { logger, loggerTagsFactory } from '@server/helpers/logger'
3import { RUNNER_JOBS } from '@server/initializers/constants' 4import { RUNNER_JOBS } from '@server/initializers/constants'
@@ -14,6 +15,8 @@ import {
14 RunnerJobSuccessPayload, 15 RunnerJobSuccessPayload,
15 RunnerJobType, 16 RunnerJobType,
16 RunnerJobUpdatePayload, 17 RunnerJobUpdatePayload,
18 RunnerJobVideoEditionTranscodingPayload,
19 RunnerJobVideoEditionTranscodingPrivatePayload,
17 RunnerJobVODAudioMergeTranscodingPayload, 20 RunnerJobVODAudioMergeTranscodingPayload,
18 RunnerJobVODAudioMergeTranscodingPrivatePayload, 21 RunnerJobVODAudioMergeTranscodingPrivatePayload,
19 RunnerJobVODHLSTranscodingPayload, 22 RunnerJobVODHLSTranscodingPayload,
@@ -21,7 +24,6 @@ import {
21 RunnerJobVODWebVideoTranscodingPayload, 24 RunnerJobVODWebVideoTranscodingPayload,
22 RunnerJobVODWebVideoTranscodingPrivatePayload 25 RunnerJobVODWebVideoTranscodingPrivatePayload
23} from '@shared/models' 26} from '@shared/models'
24import { throttle } from 'lodash'
25 27
26type CreateRunnerJobArg = 28type CreateRunnerJobArg =
27 { 29 {
@@ -43,6 +45,11 @@ type CreateRunnerJobArg =
43 type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'> 45 type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'>
44 payload: RunnerJobLiveRTMPHLSTranscodingPayload 46 payload: RunnerJobLiveRTMPHLSTranscodingPayload
45 privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload 47 privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload
48 } |
49 {
50 type: Extract<RunnerJobType, 'video-edition-transcoding'>
51 payload: RunnerJobVideoEditionTranscodingPayload
52 privatePayload: RunnerJobVideoEditionTranscodingPrivatePayload
46 } 53 }
47 54
48export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> { 55export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {
@@ -62,6 +69,8 @@ export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S
62 }): Promise<MRunnerJob> { 69 }): Promise<MRunnerJob> {
63 const { priority, dependsOnRunnerJob } = options 70 const { priority, dependsOnRunnerJob } = options
64 71
72 logger.debug('Creating runner job', { options, ...this.lTags(options.type) })
73
65 const runnerJob = new RunnerJobModel({ 74 const runnerJob = new RunnerJobModel({
66 ...pick(options, [ 'type', 'payload', 'privatePayload' ]), 75 ...pick(options, [ 'type', 'payload', 'privatePayload' ]),
67 76
diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts
index 517645848..a910ae383 100644
--- a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts
+++ b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts
@@ -4,27 +4,19 @@ import { logger } from '@server/helpers/logger'
4import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' 4import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state'
5import { VideoJobInfoModel } from '@server/models/video/video-job-info' 5import { VideoJobInfoModel } from '@server/models/video/video-job-info'
6import { MRunnerJob } from '@server/types/models/runners' 6import { MRunnerJob } from '@server/types/models/runners'
7import { 7import { RunnerJobSuccessPayload, RunnerJobUpdatePayload, RunnerJobVODPrivatePayload } from '@shared/models'
8 LiveRTMPHLSTranscodingUpdatePayload,
9 RunnerJobSuccessPayload,
10 RunnerJobUpdatePayload,
11 RunnerJobVODPrivatePayload
12} from '@shared/models'
13import { AbstractJobHandler } from './abstract-job-handler' 8import { AbstractJobHandler } from './abstract-job-handler'
14import { loadTranscodingRunnerVideo } from './shared' 9import { loadTranscodingRunnerVideo } from './shared'
15 10
16// eslint-disable-next-line max-len 11// eslint-disable-next-line max-len
17export abstract class AbstractVODTranscodingJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> extends AbstractJobHandler<C, U, S> { 12export abstract class AbstractVODTranscodingJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> extends AbstractJobHandler<C, U, S> {
18 13
19 // ---------------------------------------------------------------------------
20
21 protected isAbortSupported () { 14 protected isAbortSupported () {
22 return true 15 return true
23 } 16 }
24 17
25 protected specificUpdate (_options: { 18 protected specificUpdate (_options: {
26 runnerJob: MRunnerJob 19 runnerJob: MRunnerJob
27 updatePayload?: LiveRTMPHLSTranscodingUpdatePayload
28 }) { 20 }) {
29 // empty 21 // empty
30 } 22 }
diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts
index 0fca72b9a..a40cee865 100644
--- a/server/lib/runners/job-handlers/index.ts
+++ b/server/lib/runners/job-handlers/index.ts
@@ -1,6 +1,7 @@
1export * from './abstract-job-handler' 1export * from './abstract-job-handler'
2export * from './live-rtmp-hls-transcoding-job-handler' 2export * from './live-rtmp-hls-transcoding-job-handler'
3export * from './runner-job-handlers'
4export * from './video-edition-transcoding-job-handler'
3export * from './vod-audio-merge-transcoding-job-handler' 5export * from './vod-audio-merge-transcoding-job-handler'
4export * from './vod-hls-transcoding-job-handler' 6export * from './vod-hls-transcoding-job-handler'
5export * from './vod-web-video-transcoding-job-handler' 7export * from './vod-web-video-transcoding-job-handler'
6export * from './runner-job-handlers'
diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts
index c3d0e427d..48a70d891 100644
--- a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts
+++ b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts
@@ -70,7 +70,7 @@ export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateO
70 70
71 // --------------------------------------------------------------------------- 71 // ---------------------------------------------------------------------------
72 72
73 async specificUpdate (options: { 73 protected async specificUpdate (options: {
74 runnerJob: MRunnerJob 74 runnerJob: MRunnerJob
75 updatePayload: LiveRTMPHLSTranscodingUpdatePayload 75 updatePayload: LiveRTMPHLSTranscodingUpdatePayload
76 }) { 76 }) {
diff --git a/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/lib/runners/job-handlers/runner-job-handlers.ts
index 7bad1bc77..4ea6684ea 100644
--- a/server/lib/runners/job-handlers/runner-job-handlers.ts
+++ b/server/lib/runners/job-handlers/runner-job-handlers.ts
@@ -2,6 +2,7 @@ import { MRunnerJob } from '@server/types/models/runners'
2import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models' 2import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models'
3import { AbstractJobHandler } from './abstract-job-handler' 3import { AbstractJobHandler } from './abstract-job-handler'
4import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler' 4import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler'
5import { VideoEditionTranscodingJobHandler } from './video-edition-transcoding-job-handler'
5import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler' 6import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler'
6import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler' 7import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler'
7import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler' 8import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler'
@@ -10,7 +11,8 @@ const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, Run
10 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, 11 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler,
11 'vod-hls-transcoding': VODHLSTranscodingJobHandler, 12 'vod-hls-transcoding': VODHLSTranscodingJobHandler,
12 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, 13 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler,
13 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler 14 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler,
15 'video-edition-transcoding': VideoEditionTranscodingJobHandler
14} 16}
15 17
16export function getRunnerJobHandlerClass (job: MRunnerJob) { 18export function getRunnerJobHandlerClass (job: MRunnerJob) {
diff --git a/server/lib/runners/job-handlers/video-edition-transcoding-job-handler.ts b/server/lib/runners/job-handlers/video-edition-transcoding-job-handler.ts
new file mode 100644
index 000000000..39a755c48
--- /dev/null
+++ b/server/lib/runners/job-handlers/video-edition-transcoding-job-handler.ts
@@ -0,0 +1,157 @@
1
2import { basename } from 'path'
3import { logger } from '@server/helpers/logger'
4import { onVideoEditionEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio'
5import { MVideo } from '@server/types/models'
6import { MRunnerJob } from '@server/types/models/runners'
7import { buildUUID } from '@shared/extra-utils'
8import {
9 isVideoStudioTaskIntro,
10 isVideoStudioTaskOutro,
11 isVideoStudioTaskWatermark,
12 RunnerJobState,
13 RunnerJobUpdatePayload,
14 RunnerJobVideoEditionTranscodingPayload,
15 RunnerJobVideoEditionTranscodingPrivatePayload,
16 VideoEditionTranscodingSuccess,
17 VideoState,
18 VideoStudioTaskPayload
19} from '@shared/models'
20import { generateRunnerEditionTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls'
21import { AbstractJobHandler } from './abstract-job-handler'
22import { loadTranscodingRunnerVideo } from './shared'
23
24type CreateOptions = {
25 video: MVideo
26 tasks: VideoStudioTaskPayload[]
27 priority: number
28}
29
30// eslint-disable-next-line max-len
31export class VideoEditionTranscodingJobHandler extends AbstractJobHandler<CreateOptions, RunnerJobUpdatePayload, VideoEditionTranscodingSuccess> {
32
33 async create (options: CreateOptions) {
34 const { video, priority, tasks } = options
35
36 const jobUUID = buildUUID()
37 const payload: RunnerJobVideoEditionTranscodingPayload = {
38 input: {
39 videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid)
40 },
41 tasks: tasks.map(t => {
42 if (isVideoStudioTaskIntro(t) || isVideoStudioTaskOutro(t)) {
43 return {
44 ...t,
45
46 options: {
47 ...t.options,
48
49 file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file))
50 }
51 }
52 }
53
54 if (isVideoStudioTaskWatermark(t)) {
55 return {
56 ...t,
57
58 options: {
59 ...t.options,
60
61 file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file))
62 }
63 }
64 }
65
66 return t
67 })
68 }
69
70 const privatePayload: RunnerJobVideoEditionTranscodingPrivatePayload = {
71 videoUUID: video.uuid,
72 originalTasks: tasks
73 }
74
75 const job = await this.createRunnerJob({
76 type: 'video-edition-transcoding',
77 jobUUID,
78 payload,
79 privatePayload,
80 priority
81 })
82
83 return job
84 }
85
86 // ---------------------------------------------------------------------------
87
88 protected isAbortSupported () {
89 return true
90 }
91
92 protected specificUpdate (_options: {
93 runnerJob: MRunnerJob
94 }) {
95 // empty
96 }
97
98 protected specificAbort (_options: {
99 runnerJob: MRunnerJob
100 }) {
101 // empty
102 }
103
104 protected async specificComplete (options: {
105 runnerJob: MRunnerJob
106 resultPayload: VideoEditionTranscodingSuccess
107 }) {
108 const { runnerJob, resultPayload } = options
109 const privatePayload = runnerJob.privatePayload as RunnerJobVideoEditionTranscodingPrivatePayload
110
111 const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags)
112 if (!video) {
113 await safeCleanupStudioTMPFiles(privatePayload.originalTasks)
114
115 }
116
117 const videoFilePath = resultPayload.videoFile as string
118
119 await onVideoEditionEnded({ video, editionResultPath: videoFilePath, tasks: privatePayload.originalTasks })
120
121 logger.info(
122 'Runner video edition transcoding job %s for %s ended.',
123 runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid)
124 )
125 }
126
127 protected specificError (options: {
128 runnerJob: MRunnerJob
129 nextState: RunnerJobState
130 }) {
131 if (options.nextState === RunnerJobState.ERRORED) {
132 return this.specificErrorOrCancel(options)
133 }
134
135 return Promise.resolve()
136 }
137
138 protected specificCancel (options: {
139 runnerJob: MRunnerJob
140 }) {
141 return this.specificErrorOrCancel(options)
142 }
143
144 private async specificErrorOrCancel (options: {
145 runnerJob: MRunnerJob
146 }) {
147 const { runnerJob } = options
148
149 const payload = runnerJob.privatePayload as RunnerJobVideoEditionTranscodingPrivatePayload
150 await safeCleanupStudioTMPFiles(payload.originalTasks)
151
152 const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags)
153 if (!video) return
154
155 return video.setNewState(VideoState.PUBLISHED, false, undefined)
156 }
157}
diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts
index a7b33f87e..5f247d792 100644
--- a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts
+++ b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts
@@ -64,7 +64,7 @@ export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJo
64 64
65 // --------------------------------------------------------------------------- 65 // ---------------------------------------------------------------------------
66 66
67 async specificComplete (options: { 67 protected async specificComplete (options: {
68 runnerJob: MRunnerJob 68 runnerJob: MRunnerJob
69 resultPayload: VODAudioMergeTranscodingSuccess 69 resultPayload: VODAudioMergeTranscodingSuccess
70 }) { 70 }) {
diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts
index 02566b9d5..cc94bcbda 100644
--- a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts
+++ b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts
@@ -71,7 +71,7 @@ export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandle
71 71
72 // --------------------------------------------------------------------------- 72 // ---------------------------------------------------------------------------
73 73
74 async specificComplete (options: { 74 protected async specificComplete (options: {
75 runnerJob: MRunnerJob 75 runnerJob: MRunnerJob
76 resultPayload: VODHLSTranscodingSuccess 76 resultPayload: VODHLSTranscodingSuccess
77 }) { 77 }) {
diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts
index 57761a7a1..663d3306e 100644
--- a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts
+++ b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts
@@ -62,7 +62,7 @@ export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobH
62 62
63 // --------------------------------------------------------------------------- 63 // ---------------------------------------------------------------------------
64 64
65 async specificComplete (options: { 65 protected async specificComplete (options: {
66 runnerJob: MRunnerJob 66 runnerJob: MRunnerJob
67 resultPayload: VODWebVideoTranscodingSuccess 67 resultPayload: VODWebVideoTranscodingSuccess
68 }) { 68 }) {
diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts
index 329fb1170..a27060b33 100644
--- a/server/lib/runners/runner-urls.ts
+++ b/server/lib/runners/runner-urls.ts
@@ -7,3 +7,7 @@ export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, vid
7export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { 7export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) {
8 return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' 8 return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality'
9} 9}
10
11export function generateRunnerEditionTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string, filename: string) {
12 return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/studio/task-files/' + filename
13}
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts
index ba7916363..924adb337 100644
--- a/server/lib/server-config-manager.ts
+++ b/server/lib/server-config-manager.ts
@@ -166,7 +166,10 @@ class ServerConfigManager {
166 } 166 }
167 }, 167 },
168 videoStudio: { 168 videoStudio: {
169 enabled: CONFIG.VIDEO_STUDIO.ENABLED 169 enabled: CONFIG.VIDEO_STUDIO.ENABLED,
170 remoteRunners: {
171 enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED
172 }
170 }, 173 },
171 import: { 174 import: {
172 videos: { 175 videos: {
diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts
index 576e786d5..80dc05bfb 100644
--- a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts
+++ b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts
@@ -1,6 +1,4 @@
1 1
2import { JOB_PRIORITY } from '@server/initializers/constants'
3import { VideoModel } from '@server/models/video/video'
4import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' 2import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
5 3
6export abstract class AbstractJobBuilder { 4export abstract class AbstractJobBuilder {
@@ -20,20 +18,4 @@ export abstract class AbstractJobBuilder {
20 isNewVideo: boolean 18 isNewVideo: boolean
21 user: MUserId | null 19 user: MUserId | null
22 }): Promise<any> 20 }): Promise<any>
23
24 protected async getTranscodingJobPriority (options: {
25 user: MUserId
26 fallback: number
27 }) {
28 const { user, fallback } = options
29
30 if (!user) return fallback
31
32 const now = new Date()
33 const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
34
35 const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
36
37 return JOB_PRIORITY.TRANSCODING + videoUploadedByUser
38 }
39} 21}
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
index 5a9c93ee5..29ee2ca61 100644
--- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
+++ b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
@@ -16,6 +16,7 @@ import {
16 OptimizeTranscodingPayload, 16 OptimizeTranscodingPayload,
17 VideoTranscodingPayload 17 VideoTranscodingPayload
18} from '@shared/models' 18} from '@shared/models'
19import { getTranscodingJobPriority } from '../../transcoding-priority'
19import { canDoQuickTranscode } from '../../transcoding-quick-transcode' 20import { canDoQuickTranscode } from '../../transcoding-quick-transcode'
20import { computeResolutionsToTranscode } from '../../transcoding-resolutions' 21import { computeResolutionsToTranscode } from '../../transcoding-resolutions'
21import { AbstractJobBuilder } from './abstract-job-builder' 22import { AbstractJobBuilder } from './abstract-job-builder'
@@ -178,7 +179,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder {
178 179
179 return { 180 return {
180 type: 'video-transcoding' as 'video-transcoding', 181 type: 'video-transcoding' as 'video-transcoding',
181 priority: await this.getTranscodingJobPriority({ user, fallback: undefined }), 182 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }),
182 payload 183 payload
183 } 184 }
184 } 185 }
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
index 274dce21b..90b035402 100644
--- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
+++ b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
@@ -8,6 +8,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
8import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' 8import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models'
9import { MRunnerJob } from '@server/types/models/runners' 9import { MRunnerJob } from '@server/types/models/runners'
10import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' 10import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg'
11import { getTranscodingJobPriority } from '../../transcoding-priority'
11import { computeResolutionsToTranscode } from '../../transcoding-resolutions' 12import { computeResolutionsToTranscode } from '../../transcoding-resolutions'
12import { AbstractJobBuilder } from './abstract-job-builder' 13import { AbstractJobBuilder } from './abstract-job-builder'
13 14
@@ -49,7 +50,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
49 : resolution 50 : resolution
50 51
51 const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) 52 const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
52 const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) 53 const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
53 54
54 const mainRunnerJob = videoFile.isAudio() 55 const mainRunnerJob = videoFile.isAudio()
55 ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) 56 ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority })
@@ -63,7 +64,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
63 fps, 64 fps,
64 isNewVideo, 65 isNewVideo,
65 dependsOnRunnerJob: mainRunnerJob, 66 dependsOnRunnerJob: mainRunnerJob,
66 priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) 67 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
67 }) 68 })
68 } 69 }
69 70
@@ -96,7 +97,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
96 const maxResolution = Math.max(...resolutions) 97 const maxResolution = Math.max(...resolutions)
97 const { fps: inputFPS } = await video.probeMaxQualityFile() 98 const { fps: inputFPS } = await video.probeMaxQualityFile()
98 const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) 99 const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution })
99 const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) 100 const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
100 101
101 const childrenResolutions = resolutions.filter(r => r !== maxResolution) 102 const childrenResolutions = resolutions.filter(r => r !== maxResolution)
102 103
@@ -121,7 +122,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
121 isNewVideo, 122 isNewVideo,
122 deleteWebVideoFiles: false, 123 deleteWebVideoFiles: false,
123 dependsOnRunnerJob, 124 dependsOnRunnerJob,
124 priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) 125 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
125 }) 126 })
126 continue 127 continue
127 } 128 }
@@ -133,7 +134,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
133 fps, 134 fps,
134 isNewVideo, 135 isNewVideo,
135 dependsOnRunnerJob, 136 dependsOnRunnerJob,
136 priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) 137 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
137 }) 138 })
138 continue 139 continue
139 } 140 }
@@ -172,7 +173,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
172 fps, 173 fps,
173 isNewVideo, 174 isNewVideo,
174 dependsOnRunnerJob: mainRunnerJob, 175 dependsOnRunnerJob: mainRunnerJob,
175 priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) 176 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
176 }) 177 })
177 } 178 }
178 179
@@ -184,7 +185,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
184 isNewVideo, 185 isNewVideo,
185 deleteWebVideoFiles: false, 186 deleteWebVideoFiles: false,
186 dependsOnRunnerJob: mainRunnerJob, 187 dependsOnRunnerJob: mainRunnerJob,
187 priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) 188 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
188 }) 189 })
189 } 190 }
190 } 191 }
diff --git a/server/lib/transcoding/transcoding-priority.ts b/server/lib/transcoding/transcoding-priority.ts
new file mode 100644
index 000000000..82ab6f2f1
--- /dev/null
+++ b/server/lib/transcoding/transcoding-priority.ts
@@ -0,0 +1,24 @@
1import { JOB_PRIORITY } from '@server/initializers/constants'
2import { VideoModel } from '@server/models/video/video'
3import { MUserId } from '@server/types/models'
4
5export async function getTranscodingJobPriority (options: {
6 user: MUserId
7 fallback: number
8 type: 'vod' | 'studio'
9}) {
10 const { user, fallback, type } = options
11
12 if (!user) return fallback
13
14 const now = new Date()
15 const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
16
17 const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
18
19 const base = type === 'vod'
20 ? JOB_PRIORITY.TRANSCODING
21 : JOB_PRIORITY.VIDEO_STUDIO
22
23 return base + videoUploadedByUser
24}
diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts
index beda326a0..2c993faeb 100644
--- a/server/lib/video-studio.ts
+++ b/server/lib/video-studio.ts
@@ -1,19 +1,38 @@
1import { logger } from '@server/helpers/logger' 1import { move, remove } from 'fs-extra'
2import { MVideoFullLight } from '@server/types/models' 2import { join } from 'path'
3import { logger, loggerTagsFactory } from '@server/helpers/logger'
4import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent'
5import { CONFIG } from '@server/initializers/config'
6import { UserModel } from '@server/models/user/user'
7import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models'
3import { getVideoStreamDuration } from '@shared/ffmpeg' 8import { getVideoStreamDuration } from '@shared/ffmpeg'
4import { VideoStudioEditionPayload, VideoStudioTask } from '@shared/models' 9import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@shared/models'
5import { remove } from 'fs-extra' 10import { federateVideoIfNeeded } from './activitypub/videos'
11import { JobQueue } from './job-queue'
12import { VideoEditionTranscodingJobHandler } from './runners'
13import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job'
14import { getTranscodingJobPriority } from './transcoding/transcoding-priority'
15import { buildNewFile, removeHLSPlaylist, removeWebTorrentFile } from './video-file'
16import { VideoPathManager } from './video-path-manager'
6 17
7function buildTaskFileFieldname (indice: number, fieldName = 'file') { 18const lTags = loggerTagsFactory('video-edition')
19
20export function buildTaskFileFieldname (indice: number, fieldName = 'file') {
8 return `tasks[${indice}][options][${fieldName}]` 21 return `tasks[${indice}][options][${fieldName}]`
9} 22}
10 23
11function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { 24export function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') {
12 return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) 25 return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName))
13} 26}
14 27
15async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) { 28export function getStudioTaskFilePath (filename: string) {
16 for (const task of payload.tasks) { 29 return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, filename)
30}
31
32export async function safeCleanupStudioTMPFiles (tasks: VideoStudioTaskPayload[]) {
33 logger.info('Removing studio task files', { tasks, ...lTags() })
34
35 for (const task of tasks) {
17 try { 36 try {
18 if (task.name === 'add-intro' || task.name === 'add-outro') { 37 if (task.name === 'add-intro' || task.name === 'add-outro') {
19 await remove(task.options.file) 38 await remove(task.options.file)
@@ -26,7 +45,13 @@ async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) {
26 } 45 }
27} 46}
28 47
29async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) { 48// ---------------------------------------------------------------------------
49
50export async function approximateIntroOutroAdditionalSize (
51 video: MVideoFullLight,
52 tasks: VideoStudioTask[],
53 fileFinder: (i: number) => string
54) {
30 let additionalDuration = 0 55 let additionalDuration = 0
31 56
32 for (let i = 0; i < tasks.length; i++) { 57 for (let i = 0; i < tasks.length; i++) {
@@ -41,9 +66,65 @@ async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, task
41 return (video.getMaxQualityFile().size / video.duration) * additionalDuration 66 return (video.getMaxQualityFile().size / video.duration) * additionalDuration
42} 67}
43 68
44export { 69// ---------------------------------------------------------------------------
45 approximateIntroOutroAdditionalSize, 70
46 buildTaskFileFieldname, 71export async function createVideoStudioJob (options: {
47 getTaskFileFromReq, 72 video: MVideo
48 safeCleanupStudioTMPFiles 73 user: MUser
74 payload: VideoStudioEditionPayload
75}) {
76 const { video, user, payload } = options
77
78 const priority = await getTranscodingJobPriority({ user, type: 'studio', fallback: 0 })
79
80 if (CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED) {
81 await new VideoEditionTranscodingJobHandler().create({ video, tasks: payload.tasks, priority })
82 return
83 }
84
85 await JobQueue.Instance.createJob({ type: 'video-studio-edition', payload, priority })
86}
87
88export async function onVideoEditionEnded (options: {
89 editionResultPath: string
90 tasks: VideoStudioTaskPayload[]
91 video: MVideoFullLight
92}) {
93 const { video, tasks, editionResultPath } = options
94
95 const newFile = await buildNewFile({ path: editionResultPath, mode: 'web-video' })
96 newFile.videoId = video.id
97
98 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile)
99 await move(editionResultPath, outputPath)
100
101 await safeCleanupStudioTMPFiles(tasks)
102
103 await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)
104 await removeAllFiles(video, newFile)
105
106 await newFile.save()
107
108 video.duration = await getVideoStreamDuration(outputPath)
109 await video.save()
110
111 await federateVideoIfNeeded(video, false, undefined)
112
113 const user = await UserModel.loadByVideoId(video.id)
114
115 await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false })
116}
117
118// ---------------------------------------------------------------------------
119// Private
120// ---------------------------------------------------------------------------
121
122async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) {
123 await removeHLSPlaylist(video)
124
125 for (const file of video.VideoFiles) {
126 if (file.id === webTorrentFileException.id) continue
127
128 await removeWebTorrentFile(video, file.id)
129 }
49} 130}