aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/transcoding/shared/job-builders
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib/transcoding/shared/job-builders')
-rw-r--r--server/lib/transcoding/shared/job-builders/abstract-job-builder.ts21
-rw-r--r--server/lib/transcoding/shared/job-builders/index.ts2
-rw-r--r--server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts322
-rw-r--r--server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts196
4 files changed, 0 insertions, 541 deletions
diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts
deleted file mode 100644
index 15fc814ae..000000000
--- a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts
+++ /dev/null
@@ -1,21 +0,0 @@
1
2import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
3
4export abstract class AbstractJobBuilder {
5
6 abstract createOptimizeOrMergeAudioJobs (options: {
7 video: MVideoFullLight
8 videoFile: MVideoFile
9 isNewVideo: boolean
10 user: MUserId
11 videoFileAlreadyLocked: boolean
12 }): Promise<any>
13
14 abstract createTranscodingJobs (options: {
15 transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7
16 video: MVideoFullLight
17 resolutions: number[]
18 isNewVideo: boolean
19 user: MUserId | null
20 }): Promise<any>
21}
diff --git a/server/lib/transcoding/shared/job-builders/index.ts b/server/lib/transcoding/shared/job-builders/index.ts
deleted file mode 100644
index 9b1c82adf..000000000
--- a/server/lib/transcoding/shared/job-builders/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
1export * from './transcoding-job-queue-builder'
2export * from './transcoding-runner-job-builder'
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
deleted file mode 100644
index 0505c2b2f..000000000
--- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
+++ /dev/null
@@ -1,322 +0,0 @@
1import Bluebird from 'bluebird'
2import { computeOutputFPS } from '@server/helpers/ffmpeg'
3import { logger } from '@server/helpers/logger'
4import { CONFIG } from '@server/initializers/config'
5import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
6import { CreateJobArgument, JobQueue } from '@server/lib/job-queue'
7import { Hooks } from '@server/lib/plugins/hooks'
8import { VideoPathManager } from '@server/lib/video-path-manager'
9import { VideoJobInfoModel } from '@server/models/video/video-job-info'
10import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models'
11import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg'
12import {
13 HLSTranscodingPayload,
14 MergeAudioTranscodingPayload,
15 NewWebVideoResolutionTranscodingPayload,
16 OptimizeTranscodingPayload,
17 VideoTranscodingPayload
18} from '@shared/models'
19import { getTranscodingJobPriority } from '../../transcoding-priority'
20import { canDoQuickTranscode } from '../../transcoding-quick-transcode'
21import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions'
22import { AbstractJobBuilder } from './abstract-job-builder'
23
24export class TranscodingJobQueueBuilder extends AbstractJobBuilder {
25
26 async createOptimizeOrMergeAudioJobs (options: {
27 video: MVideoFullLight
28 videoFile: MVideoFile
29 isNewVideo: boolean
30 user: MUserId
31 videoFileAlreadyLocked: boolean
32 }) {
33 const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options
34
35 let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload
36 let nextTranscodingSequentialJobPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = []
37
38 const mutexReleaser = videoFileAlreadyLocked
39 ? () => {}
40 : await VideoPathManager.Instance.lockFiles(video.uuid)
41
42 try {
43 await video.reload()
44 await videoFile.reload()
45
46 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => {
47 const probe = await ffprobePromise(videoFilePath)
48
49 const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe)
50 const hasAudio = await hasAudioStream(videoFilePath, probe)
51 const quickTranscode = await canDoQuickTranscode(videoFilePath, probe)
52 const inputFPS = videoFile.isAudio()
53 ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value
54 : await getVideoStreamFPS(videoFilePath, probe)
55
56 const maxResolution = await isAudioFile(videoFilePath, probe)
57 ? DEFAULT_AUDIO_RESOLUTION
58 : buildOriginalFileResolution(resolution)
59
60 if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
61 nextTranscodingSequentialJobPayloads.push([
62 this.buildHLSJobPayload({
63 deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false,
64
65 // We had some issues with a web video quick transcoded while producing a HLS version of it
66 copyCodecs: !quickTranscode,
67
68 resolution: maxResolution,
69 fps: computeOutputFPS({ inputFPS, resolution: maxResolution }),
70 videoUUID: video.uuid,
71 isNewVideo
72 })
73 ])
74 }
75
76 const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({
77 video,
78 inputVideoResolution: maxResolution,
79 inputVideoFPS: inputFPS,
80 hasAudio,
81 isNewVideo
82 })
83
84 nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ]
85
86 const hasChildren = nextTranscodingSequentialJobPayloads.length !== 0
87 mergeOrOptimizePayload = videoFile.isAudio()
88 ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo, hasChildren })
89 : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode, hasChildren })
90 })
91 } finally {
92 mutexReleaser()
93 }
94
95 const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => {
96 return Bluebird.mapSeries(payloads, payload => {
97 return this.buildTranscodingJob({ payload, user })
98 })
99 })
100
101 const transcodingJobBuilderJob: CreateJobArgument = {
102 type: 'transcoding-job-builder',
103 payload: {
104 videoUUID: video.uuid,
105 sequentialJobs: nextTranscodingSequentialJobs
106 }
107 }
108
109 const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user })
110
111 await JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ])
112
113 await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
114 }
115
116 // ---------------------------------------------------------------------------
117
118 async createTranscodingJobs (options: {
119 transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7
120 video: MVideoFullLight
121 resolutions: number[]
122 isNewVideo: boolean
123 user: MUserId | null
124 }) {
125 const { video, transcodingType, resolutions, isNewVideo } = options
126
127 const maxResolution = Math.max(...resolutions)
128 const childrenResolutions = resolutions.filter(r => r !== maxResolution)
129
130 logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution })
131
132 const { fps: inputFPS } = await video.probeMaxQualityFile()
133
134 const children = childrenResolutions.map(resolution => {
135 const fps = computeOutputFPS({ inputFPS, resolution })
136
137 if (transcodingType === 'hls') {
138 return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo })
139 }
140
141 if (transcodingType === 'webtorrent' || transcodingType === 'web-video') {
142 return this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo })
143 }
144
145 throw new Error('Unknown transcoding type')
146 })
147
148 const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
149
150 const parent = transcodingType === 'hls'
151 ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo })
152 : this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo })
153
154 // Process the last resolution after the other ones to prevent concurrency issue
155 // Because low resolutions use the biggest one as ffmpeg input
156 await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null })
157 }
158
159 // ---------------------------------------------------------------------------
160
161 private async createTranscodingJobsWithChildren (options: {
162 videoUUID: string
163 parent: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload)
164 children: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload)[]
165 user: MUserId | null
166 }) {
167 const { videoUUID, parent, children, user } = options
168
169 const parentJob = await this.buildTranscodingJob({ payload: parent, user })
170 const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user }))
171
172 await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs)
173
174 await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length)
175 }
176
177 private async buildTranscodingJob (options: {
178 payload: VideoTranscodingPayload
179 user: MUserId | null // null means we don't want priority
180 }) {
181 const { user, payload } = options
182
183 return {
184 type: 'video-transcoding' as 'video-transcoding',
185 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }),
186 payload
187 }
188 }
189
190 private async buildLowerResolutionJobPayloads (options: {
191 video: MVideoWithFileThumbnail
192 inputVideoResolution: number
193 inputVideoFPS: number
194 hasAudio: boolean
195 isNewVideo: boolean
196 }) {
197 const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options
198
199 // Create transcoding jobs if there are enabled resolutions
200 const resolutionsEnabled = await Hooks.wrapObject(
201 computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
202 'filter:transcoding.auto.resolutions-to-transcode.result',
203 options
204 )
205
206 const sequentialPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = []
207
208 for (const resolution of resolutionsEnabled) {
209 const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution })
210
211 if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) {
212 const payloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[] = [
213 this.buildWebVideoJobPayload({
214 videoUUID: video.uuid,
215 resolution,
216 fps,
217 isNewVideo
218 })
219 ]
220
221 // Create a subsequent job to create HLS resolution that will just copy web video codecs
222 if (CONFIG.TRANSCODING.HLS.ENABLED) {
223 payloads.push(
224 this.buildHLSJobPayload({
225 videoUUID: video.uuid,
226 resolution,
227 fps,
228 isNewVideo,
229 copyCodecs: true
230 })
231 )
232 }
233
234 sequentialPayloads.push(payloads)
235 } else if (CONFIG.TRANSCODING.HLS.ENABLED) {
236 sequentialPayloads.push([
237 this.buildHLSJobPayload({
238 videoUUID: video.uuid,
239 resolution,
240 fps,
241 copyCodecs: false,
242 isNewVideo
243 })
244 ])
245 }
246 }
247
248 return sequentialPayloads
249 }
250
251 private buildHLSJobPayload (options: {
252 videoUUID: string
253 resolution: number
254 fps: number
255 isNewVideo: boolean
256 deleteWebVideoFiles?: boolean // default false
257 copyCodecs?: boolean // default false
258 }): HLSTranscodingPayload {
259 const { videoUUID, resolution, fps, isNewVideo, deleteWebVideoFiles = false, copyCodecs = false } = options
260
261 return {
262 type: 'new-resolution-to-hls',
263 videoUUID,
264 resolution,
265 fps,
266 copyCodecs,
267 isNewVideo,
268 deleteWebVideoFiles
269 }
270 }
271
272 private buildWebVideoJobPayload (options: {
273 videoUUID: string
274 resolution: number
275 fps: number
276 isNewVideo: boolean
277 }): NewWebVideoResolutionTranscodingPayload {
278 const { videoUUID, resolution, fps, isNewVideo } = options
279
280 return {
281 type: 'new-resolution-to-web-video',
282 videoUUID,
283 isNewVideo,
284 resolution,
285 fps
286 }
287 }
288
289 private buildMergeAudioPayload (options: {
290 videoUUID: string
291 isNewVideo: boolean
292 hasChildren: boolean
293 }): MergeAudioTranscodingPayload {
294 const { videoUUID, isNewVideo, hasChildren } = options
295
296 return {
297 type: 'merge-audio-to-web-video',
298 resolution: DEFAULT_AUDIO_RESOLUTION,
299 fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE,
300 videoUUID,
301 isNewVideo,
302 hasChildren
303 }
304 }
305
306 private buildOptimizePayload (options: {
307 videoUUID: string
308 quickTranscode: boolean
309 isNewVideo: boolean
310 hasChildren: boolean
311 }): OptimizeTranscodingPayload {
312 const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options
313
314 return {
315 type: 'optimize-to-web-video',
316 videoUUID,
317 isNewVideo,
318 hasChildren,
319 quickTranscode
320 }
321 }
322}
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
deleted file mode 100644
index f0671bd7a..000000000
--- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
+++ /dev/null
@@ -1,196 +0,0 @@
1import { computeOutputFPS } from '@server/helpers/ffmpeg'
2import { logger, loggerTagsFactory } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
5import { Hooks } from '@server/lib/plugins/hooks'
6import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners'
7import { VideoPathManager } from '@server/lib/video-path-manager'
8import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models'
9import { MRunnerJob } from '@server/types/models/runners'
10import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg'
11import { getTranscodingJobPriority } from '../../transcoding-priority'
12import { computeResolutionsToTranscode } from '../../transcoding-resolutions'
13import { AbstractJobBuilder } from './abstract-job-builder'
14
15/**
16 *
17 * Class to build transcoding job in the local job queue
18 *
19 */
20
21const lTags = loggerTagsFactory('transcoding')
22
23export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
24
25 async createOptimizeOrMergeAudioJobs (options: {
26 video: MVideoFullLight
27 videoFile: MVideoFile
28 isNewVideo: boolean
29 user: MUserId
30 videoFileAlreadyLocked: boolean
31 }) {
32 const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options
33
34 const mutexReleaser = videoFileAlreadyLocked
35 ? () => {}
36 : await VideoPathManager.Instance.lockFiles(video.uuid)
37
38 try {
39 await video.reload()
40 await videoFile.reload()
41
42 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => {
43 const probe = await ffprobePromise(videoFilePath)
44
45 const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe)
46 const hasAudio = await hasAudioStream(videoFilePath, probe)
47 const inputFPS = videoFile.isAudio()
48 ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value
49 : await getVideoStreamFPS(videoFilePath, probe)
50
51 const maxResolution = await isAudioFile(videoFilePath, probe)
52 ? DEFAULT_AUDIO_RESOLUTION
53 : resolution
54
55 const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
56 const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
57
58 const mainRunnerJob = videoFile.isAudio()
59 ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority })
60 : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority })
61
62 if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
63 await new VODHLSTranscodingJobHandler().create({
64 video,
65 deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false,
66 resolution: maxResolution,
67 fps,
68 isNewVideo,
69 dependsOnRunnerJob: mainRunnerJob,
70 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
71 })
72 }
73
74 await this.buildLowerResolutionJobPayloads({
75 video,
76 inputVideoResolution: maxResolution,
77 inputVideoFPS: inputFPS,
78 hasAudio,
79 isNewVideo,
80 mainRunnerJob,
81 user
82 })
83 })
84 } finally {
85 mutexReleaser()
86 }
87 }
88
89 // ---------------------------------------------------------------------------
90
91 async createTranscodingJobs (options: {
92 transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7
93 video: MVideoFullLight
94 resolutions: number[]
95 isNewVideo: boolean
96 user: MUserId | null
97 }) {
98 const { video, transcodingType, resolutions, isNewVideo, user } = options
99
100 const maxResolution = Math.max(...resolutions)
101 const { fps: inputFPS } = await video.probeMaxQualityFile()
102 const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution })
103 const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
104
105 const childrenResolutions = resolutions.filter(r => r !== maxResolution)
106
107 logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution })
108
109 // Process the last resolution before the other ones to prevent concurrency issue
110 // Because low resolutions use the biggest one as ffmpeg input
111 const mainJob = transcodingType === 'hls'
112 // eslint-disable-next-line max-len
113 ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority })
114 : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority })
115
116 for (const resolution of childrenResolutions) {
117 const dependsOnRunnerJob = mainJob
118 const fps = computeOutputFPS({ inputFPS, resolution })
119
120 if (transcodingType === 'hls') {
121 await new VODHLSTranscodingJobHandler().create({
122 video,
123 resolution,
124 fps,
125 isNewVideo,
126 deleteWebVideoFiles: false,
127 dependsOnRunnerJob,
128 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
129 })
130 continue
131 }
132
133 if (transcodingType === 'webtorrent' || transcodingType === 'web-video') {
134 await new VODWebVideoTranscodingJobHandler().create({
135 video,
136 resolution,
137 fps,
138 isNewVideo,
139 dependsOnRunnerJob,
140 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
141 })
142 continue
143 }
144
145 throw new Error('Unknown transcoding type')
146 }
147 }
148
149 private async buildLowerResolutionJobPayloads (options: {
150 mainRunnerJob: MRunnerJob
151 video: MVideoWithFileThumbnail
152 inputVideoResolution: number
153 inputVideoFPS: number
154 hasAudio: boolean
155 isNewVideo: boolean
156 user: MUserId
157 }) {
158 const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options
159
160 // Create transcoding jobs if there are enabled resolutions
161 const resolutionsEnabled = await Hooks.wrapObject(
162 computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
163 'filter:transcoding.auto.resolutions-to-transcode.result',
164 options
165 )
166
167 logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) })
168
169 for (const resolution of resolutionsEnabled) {
170 const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution })
171
172 if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) {
173 await new VODWebVideoTranscodingJobHandler().create({
174 video,
175 resolution,
176 fps,
177 isNewVideo,
178 dependsOnRunnerJob: mainRunnerJob,
179 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
180 })
181 }
182
183 if (CONFIG.TRANSCODING.HLS.ENABLED) {
184 await new VODHLSTranscodingJobHandler().create({
185 video,
186 resolution,
187 fps,
188 isNewVideo,
189 deleteWebVideoFiles: false,
190 dependsOnRunnerJob: mainRunnerJob,
191 priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
192 })
193 }
194 }
195 }
196}
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-16 07:09:45 +0000