aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
blob: c09c6b8ce9b6cfc56deaa86be31b64dbcc4a7b9b (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
import Bluebird from 'bluebird'
import { computeOutputFPS } from '@server/helpers/ffmpeg'
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
import { CreateJobArgument, JobQueue } from '@server/lib/job-queue'
import { Hooks } from '@server/lib/plugins/hooks'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models'
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg'
import {
  HLSTranscodingPayload,
  MergeAudioTranscodingPayload,
  NewWebTorrentResolutionTranscodingPayload,
  OptimizeTranscodingPayload,
  VideoTranscodingPayload
} from '@shared/models'
import { canDoQuickTranscode } from '../../transcoding-quick-transcode'
import { computeResolutionsToTranscode } from '../../transcoding-resolutions'
import { AbstractJobBuilder } from './abstract-job-builder'

export class TranscodingJobQueueBuilder extends AbstractJobBuilder {

  async createOptimizeOrMergeAudioJobs (options: {
    video: MVideoFullLight
    videoFile: MVideoFile
    isNewVideo: boolean
    user: MUserId
  }) {
    const { video, videoFile, isNewVideo, user } = options

    let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload
    let nextTranscodingSequentialJobPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = []

    const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)

    try {
      await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => {
        const probe = await ffprobePromise(videoFilePath)

        const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe)
        const hasAudio = await hasAudioStream(videoFilePath, probe)
        const quickTranscode = await canDoQuickTranscode(videoFilePath, probe)
        const inputFPS = videoFile.isAudio()
          ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value
          : await getVideoStreamFPS(videoFilePath, probe)

        const maxResolution = await isAudioFile(videoFilePath, probe)
          ? DEFAULT_AUDIO_RESOLUTION
          : resolution

        if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
          nextTranscodingSequentialJobPayloads.push([
            this.buildHLSJobPayload({
              deleteWebTorrentFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false,

              // We had some issues with a web video quick transcoded while producing a HLS version of it
              copyCodecs: !quickTranscode,

              resolution: maxResolution,
              fps: computeOutputFPS({ inputFPS, resolution: maxResolution }),
              videoUUID: video.uuid,
              isNewVideo
            })
          ])
        }

        const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({
          video,
          inputVideoResolution: maxResolution,
          inputVideoFPS: inputFPS,
          hasAudio,
          isNewVideo
        })

        nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ]

        const hasChildren = nextTranscodingSequentialJobPayloads.length !== 0
        mergeOrOptimizePayload = videoFile.isAudio()
          ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo, hasChildren })
          : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode, hasChildren })
      })
    } finally {
      mutexReleaser()
    }

    const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => {
      return Bluebird.mapSeries(payloads, payload => {
        return this.buildTranscodingJob({ payload, user })
      })
    })

    const transcodingJobBuilderJob: CreateJobArgument = {
      type: 'transcoding-job-builder',
      payload: {
        videoUUID: video.uuid,
        sequentialJobs: nextTranscodingSequentialJobs
      }
    }

    const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user })

    await JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ])

    await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
  }

  // ---------------------------------------------------------------------------

  async createTranscodingJobs (options: {
    transcodingType: 'hls' | 'webtorrent'
    video: MVideoFullLight
    resolutions: number[]
    isNewVideo: boolean
    user: MUserId | null
  }) {
    const { video, transcodingType, resolutions, isNewVideo } = options

    const maxResolution = Math.max(...resolutions)
    const childrenResolutions = resolutions.filter(r => r !== maxResolution)

    logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution })

    const { fps: inputFPS } = await video.probeMaxQualityFile()

    const children = childrenResolutions.map(resolution => {
      const fps = computeOutputFPS({ inputFPS, resolution })

      if (transcodingType === 'hls') {
        return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo })
      }

      if (transcodingType === 'webtorrent') {
        return this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo })
      }

      throw new Error('Unknown transcoding type')
    })

    const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })

    const parent = transcodingType === 'hls'
      ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo })
      : this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo })

    // Process the last resolution after the other ones to prevent concurrency issue
    // Because low resolutions use the biggest one as ffmpeg input
    await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null })
  }

  // ---------------------------------------------------------------------------

  private async createTranscodingJobsWithChildren (options: {
    videoUUID: string
    parent: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)
    children: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)[]
    user: MUserId | null
  }) {
    const { videoUUID, parent, children, user } = options

    const parentJob = await this.buildTranscodingJob({ payload: parent, user })
    const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user }))

    await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs)

    await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length)
  }

  private async buildTranscodingJob (options: {
    payload: VideoTranscodingPayload
    user: MUserId | null // null means we don't want priority
  }) {
    const { user, payload } = options

    return {
      type: 'video-transcoding' as 'video-transcoding',
      priority: await this.getTranscodingJobPriority({ user, fallback: undefined }),
      payload
    }
  }

  private async buildLowerResolutionJobPayloads (options: {
    video: MVideoWithFileThumbnail
    inputVideoResolution: number
    inputVideoFPS: number
    hasAudio: boolean
    isNewVideo: boolean
  }) {
    const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options

    // Create transcoding jobs if there are enabled resolutions
    const resolutionsEnabled = await Hooks.wrapObject(
      computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
      'filter:transcoding.auto.resolutions-to-transcode.result',
      options
    )

    const sequentialPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = []

    for (const resolution of resolutionsEnabled) {
      const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution })

      if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) {
        const payloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[] = [
          this.buildWebTorrentJobPayload({
            videoUUID: video.uuid,
            resolution,
            fps,
            isNewVideo
          })
        ]

        // Create a subsequent job to create HLS resolution that will just copy web video codecs
        if (CONFIG.TRANSCODING.HLS.ENABLED) {
          payloads.push(
            this.buildHLSJobPayload({
              videoUUID: video.uuid,
              resolution,
              fps,
              isNewVideo,
              copyCodecs: true
            })
          )
        }

        sequentialPayloads.push(payloads)
      } else if (CONFIG.TRANSCODING.HLS.ENABLED) {
        sequentialPayloads.push([
          this.buildHLSJobPayload({
            videoUUID: video.uuid,
            resolution,
            fps,
            copyCodecs: false,
            isNewVideo
          })
        ])
      }
    }

    return sequentialPayloads
  }

  private buildHLSJobPayload (options: {
    videoUUID: string
    resolution: number
    fps: number
    isNewVideo: boolean
    deleteWebTorrentFiles?: boolean // default false
    copyCodecs?: boolean // default false
  }): HLSTranscodingPayload {
    const { videoUUID, resolution, fps, isNewVideo, deleteWebTorrentFiles = false, copyCodecs = false } = options

    return {
      type: 'new-resolution-to-hls',
      videoUUID,
      resolution,
      fps,
      copyCodecs,
      isNewVideo,
      deleteWebTorrentFiles
    }
  }

  private buildWebTorrentJobPayload (options: {
    videoUUID: string
    resolution: number
    fps: number
    isNewVideo: boolean
  }): NewWebTorrentResolutionTranscodingPayload {
    const { videoUUID, resolution, fps, isNewVideo } = options

    return {
      type: 'new-resolution-to-webtorrent',
      videoUUID,
      isNewVideo,
      resolution,
      fps
    }
  }

  private buildMergeAudioPayload (options: {
    videoUUID: string
    isNewVideo: boolean
    hasChildren: boolean
  }): MergeAudioTranscodingPayload {
    const { videoUUID, isNewVideo, hasChildren } = options

    return {
      type: 'merge-audio-to-webtorrent',
      resolution: DEFAULT_AUDIO_RESOLUTION,
      fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE,
      videoUUID,
      isNewVideo,
      hasChildren
    }
  }

  private buildOptimizePayload (options: {
    videoUUID: string
    quickTranscode: boolean
    isNewVideo: boolean
    hasChildren: boolean
  }): OptimizeTranscodingPayload {
    const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options

    return {
      type: 'optimize-to-webtorrent',
      videoUUID,
      isNewVideo,
      hasChildren,
      quickTranscode
    }
  }
}