diff options
Diffstat (limited to 'server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts')
-rw-r--r-- | server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts | 322 |
1 files changed, 0 insertions, 322 deletions
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts deleted file mode 100644 index 0505c2b2f..000000000 --- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts +++ /dev/null | |||
@@ -1,322 +0,0 @@ | |||
1 | import Bluebird from 'bluebird' | ||
2 | import { computeOutputFPS } from '@server/helpers/ffmpeg' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
6 | import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' | ||
7 | import { Hooks } from '@server/lib/plugins/hooks' | ||
8 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
10 | import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' | ||
11 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' | ||
12 | import { | ||
13 | HLSTranscodingPayload, | ||
14 | MergeAudioTranscodingPayload, | ||
15 | NewWebVideoResolutionTranscodingPayload, | ||
16 | OptimizeTranscodingPayload, | ||
17 | VideoTranscodingPayload | ||
18 | } from '@shared/models' | ||
19 | import { getTranscodingJobPriority } from '../../transcoding-priority' | ||
20 | import { canDoQuickTranscode } from '../../transcoding-quick-transcode' | ||
21 | import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions' | ||
22 | import { AbstractJobBuilder } from './abstract-job-builder' | ||
23 | |||
24 | export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | ||
25 | |||
26 | async createOptimizeOrMergeAudioJobs (options: { | ||
27 | video: MVideoFullLight | ||
28 | videoFile: MVideoFile | ||
29 | isNewVideo: boolean | ||
30 | user: MUserId | ||
31 | videoFileAlreadyLocked: boolean | ||
32 | }) { | ||
33 | const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options | ||
34 | |||
35 | let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload | ||
36 | let nextTranscodingSequentialJobPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] | ||
37 | |||
38 | const mutexReleaser = videoFileAlreadyLocked | ||
39 | ? () => {} | ||
40 | : await VideoPathManager.Instance.lockFiles(video.uuid) | ||
41 | |||
42 | try { | ||
43 | await video.reload() | ||
44 | await videoFile.reload() | ||
45 | |||
46 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { | ||
47 | const probe = await ffprobePromise(videoFilePath) | ||
48 | |||
49 | const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) | ||
50 | const hasAudio = await hasAudioStream(videoFilePath, probe) | ||
51 | const quickTranscode = await canDoQuickTranscode(videoFilePath, probe) | ||
52 | const inputFPS = videoFile.isAudio() | ||
53 | ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value | ||
54 | : await getVideoStreamFPS(videoFilePath, probe) | ||
55 | |||
56 | const maxResolution = await isAudioFile(videoFilePath, probe) | ||
57 | ? DEFAULT_AUDIO_RESOLUTION | ||
58 | : buildOriginalFileResolution(resolution) | ||
59 | |||
60 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { | ||
61 | nextTranscodingSequentialJobPayloads.push([ | ||
62 | this.buildHLSJobPayload({ | ||
63 | deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, | ||
64 | |||
65 | // We had some issues with a web video quick transcoded while producing a HLS version of it | ||
66 | copyCodecs: !quickTranscode, | ||
67 | |||
68 | resolution: maxResolution, | ||
69 | fps: computeOutputFPS({ inputFPS, resolution: maxResolution }), | ||
70 | videoUUID: video.uuid, | ||
71 | isNewVideo | ||
72 | }) | ||
73 | ]) | ||
74 | } | ||
75 | |||
76 | const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({ | ||
77 | video, | ||
78 | inputVideoResolution: maxResolution, | ||
79 | inputVideoFPS: inputFPS, | ||
80 | hasAudio, | ||
81 | isNewVideo | ||
82 | }) | ||
83 | |||
84 | nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ] | ||
85 | |||
86 | const hasChildren = nextTranscodingSequentialJobPayloads.length !== 0 | ||
87 | mergeOrOptimizePayload = videoFile.isAudio() | ||
88 | ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo, hasChildren }) | ||
89 | : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode, hasChildren }) | ||
90 | }) | ||
91 | } finally { | ||
92 | mutexReleaser() | ||
93 | } | ||
94 | |||
95 | const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => { | ||
96 | return Bluebird.mapSeries(payloads, payload => { | ||
97 | return this.buildTranscodingJob({ payload, user }) | ||
98 | }) | ||
99 | }) | ||
100 | |||
101 | const transcodingJobBuilderJob: CreateJobArgument = { | ||
102 | type: 'transcoding-job-builder', | ||
103 | payload: { | ||
104 | videoUUID: video.uuid, | ||
105 | sequentialJobs: nextTranscodingSequentialJobs | ||
106 | } | ||
107 | } | ||
108 | |||
109 | const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user }) | ||
110 | |||
111 | await JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ]) | ||
112 | |||
113 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') | ||
114 | } | ||
115 | |||
116 | // --------------------------------------------------------------------------- | ||
117 | |||
118 | async createTranscodingJobs (options: { | ||
119 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 | ||
120 | video: MVideoFullLight | ||
121 | resolutions: number[] | ||
122 | isNewVideo: boolean | ||
123 | user: MUserId | null | ||
124 | }) { | ||
125 | const { video, transcodingType, resolutions, isNewVideo } = options | ||
126 | |||
127 | const maxResolution = Math.max(...resolutions) | ||
128 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) | ||
129 | |||
130 | logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) | ||
131 | |||
132 | const { fps: inputFPS } = await video.probeMaxQualityFile() | ||
133 | |||
134 | const children = childrenResolutions.map(resolution => { | ||
135 | const fps = computeOutputFPS({ inputFPS, resolution }) | ||
136 | |||
137 | if (transcodingType === 'hls') { | ||
138 | return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) | ||
139 | } | ||
140 | |||
141 | if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { | ||
142 | return this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) | ||
143 | } | ||
144 | |||
145 | throw new Error('Unknown transcoding type') | ||
146 | }) | ||
147 | |||
148 | const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) | ||
149 | |||
150 | const parent = transcodingType === 'hls' | ||
151 | ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) | ||
152 | : this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) | ||
153 | |||
154 | // Process the last resolution after the other ones to prevent concurrency issue | ||
155 | // Because low resolutions use the biggest one as ffmpeg input | ||
156 | await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null }) | ||
157 | } | ||
158 | |||
159 | // --------------------------------------------------------------------------- | ||
160 | |||
161 | private async createTranscodingJobsWithChildren (options: { | ||
162 | videoUUID: string | ||
163 | parent: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload) | ||
164 | children: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload)[] | ||
165 | user: MUserId | null | ||
166 | }) { | ||
167 | const { videoUUID, parent, children, user } = options | ||
168 | |||
169 | const parentJob = await this.buildTranscodingJob({ payload: parent, user }) | ||
170 | const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user })) | ||
171 | |||
172 | await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs) | ||
173 | |||
174 | await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length) | ||
175 | } | ||
176 | |||
177 | private async buildTranscodingJob (options: { | ||
178 | payload: VideoTranscodingPayload | ||
179 | user: MUserId | null // null means we don't want priority | ||
180 | }) { | ||
181 | const { user, payload } = options | ||
182 | |||
183 | return { | ||
184 | type: 'video-transcoding' as 'video-transcoding', | ||
185 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }), | ||
186 | payload | ||
187 | } | ||
188 | } | ||
189 | |||
190 | private async buildLowerResolutionJobPayloads (options: { | ||
191 | video: MVideoWithFileThumbnail | ||
192 | inputVideoResolution: number | ||
193 | inputVideoFPS: number | ||
194 | hasAudio: boolean | ||
195 | isNewVideo: boolean | ||
196 | }) { | ||
197 | const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options | ||
198 | |||
199 | // Create transcoding jobs if there are enabled resolutions | ||
200 | const resolutionsEnabled = await Hooks.wrapObject( | ||
201 | computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), | ||
202 | 'filter:transcoding.auto.resolutions-to-transcode.result', | ||
203 | options | ||
204 | ) | ||
205 | |||
206 | const sequentialPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] | ||
207 | |||
208 | for (const resolution of resolutionsEnabled) { | ||
209 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) | ||
210 | |||
211 | if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { | ||
212 | const payloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ | ||
213 | this.buildWebVideoJobPayload({ | ||
214 | videoUUID: video.uuid, | ||
215 | resolution, | ||
216 | fps, | ||
217 | isNewVideo | ||
218 | }) | ||
219 | ] | ||
220 | |||
221 | // Create a subsequent job to create HLS resolution that will just copy web video codecs | ||
222 | if (CONFIG.TRANSCODING.HLS.ENABLED) { | ||
223 | payloads.push( | ||
224 | this.buildHLSJobPayload({ | ||
225 | videoUUID: video.uuid, | ||
226 | resolution, | ||
227 | fps, | ||
228 | isNewVideo, | ||
229 | copyCodecs: true | ||
230 | }) | ||
231 | ) | ||
232 | } | ||
233 | |||
234 | sequentialPayloads.push(payloads) | ||
235 | } else if (CONFIG.TRANSCODING.HLS.ENABLED) { | ||
236 | sequentialPayloads.push([ | ||
237 | this.buildHLSJobPayload({ | ||
238 | videoUUID: video.uuid, | ||
239 | resolution, | ||
240 | fps, | ||
241 | copyCodecs: false, | ||
242 | isNewVideo | ||
243 | }) | ||
244 | ]) | ||
245 | } | ||
246 | } | ||
247 | |||
248 | return sequentialPayloads | ||
249 | } | ||
250 | |||
251 | private buildHLSJobPayload (options: { | ||
252 | videoUUID: string | ||
253 | resolution: number | ||
254 | fps: number | ||
255 | isNewVideo: boolean | ||
256 | deleteWebVideoFiles?: boolean // default false | ||
257 | copyCodecs?: boolean // default false | ||
258 | }): HLSTranscodingPayload { | ||
259 | const { videoUUID, resolution, fps, isNewVideo, deleteWebVideoFiles = false, copyCodecs = false } = options | ||
260 | |||
261 | return { | ||
262 | type: 'new-resolution-to-hls', | ||
263 | videoUUID, | ||
264 | resolution, | ||
265 | fps, | ||
266 | copyCodecs, | ||
267 | isNewVideo, | ||
268 | deleteWebVideoFiles | ||
269 | } | ||
270 | } | ||
271 | |||
272 | private buildWebVideoJobPayload (options: { | ||
273 | videoUUID: string | ||
274 | resolution: number | ||
275 | fps: number | ||
276 | isNewVideo: boolean | ||
277 | }): NewWebVideoResolutionTranscodingPayload { | ||
278 | const { videoUUID, resolution, fps, isNewVideo } = options | ||
279 | |||
280 | return { | ||
281 | type: 'new-resolution-to-web-video', | ||
282 | videoUUID, | ||
283 | isNewVideo, | ||
284 | resolution, | ||
285 | fps | ||
286 | } | ||
287 | } | ||
288 | |||
289 | private buildMergeAudioPayload (options: { | ||
290 | videoUUID: string | ||
291 | isNewVideo: boolean | ||
292 | hasChildren: boolean | ||
293 | }): MergeAudioTranscodingPayload { | ||
294 | const { videoUUID, isNewVideo, hasChildren } = options | ||
295 | |||
296 | return { | ||
297 | type: 'merge-audio-to-web-video', | ||
298 | resolution: DEFAULT_AUDIO_RESOLUTION, | ||
299 | fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, | ||
300 | videoUUID, | ||
301 | isNewVideo, | ||
302 | hasChildren | ||
303 | } | ||
304 | } | ||
305 | |||
306 | private buildOptimizePayload (options: { | ||
307 | videoUUID: string | ||
308 | quickTranscode: boolean | ||
309 | isNewVideo: boolean | ||
310 | hasChildren: boolean | ||
311 | }): OptimizeTranscodingPayload { | ||
312 | const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options | ||
313 | |||
314 | return { | ||
315 | type: 'optimize-to-web-video', | ||
316 | videoUUID, | ||
317 | isNewVideo, | ||
318 | hasChildren, | ||
319 | quickTranscode | ||
320 | } | ||
321 | } | ||
322 | } | ||