1 import { MutexInterface } from 'async-mutex'
2 import { Job } from 'bullmq'
3 import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
4 import { basename, extname as extnameUtil, join } from 'path'
5 import { toEven } from '@server/helpers/core-utils'
6 import { retryTransactionWrapper } from '@server/helpers/database-utils'
7 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
8 import { sequelizeTypescript } from '@server/initializers/database'
9 import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
10 import { pick } from '@shared/core-utils'
11 import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
15 computeResolutionsToTranscode,
17 getVideoStreamDuration,
21 TranscodeVODOptionsType
22 } from '../../helpers/ffmpeg'
23 import { CONFIG } from '../../initializers/config'
24 import { VideoFileModel } from '../../models/video/video-file'
25 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
26 import { updatePlaylistAfterFileChange } from '../hls'
27 import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths'
28 import { VideoPathManager } from '../video-path-manager'
29 import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
33 * Functions that run transcoding functions, update the database, cleanup files, create torrent files...
34 * Mainly called by the job queue
38 // Optimize the original video file and replace it. The resolution is not changed.
39 async function optimizeOriginalVideofile (options: {
40 video: MVideoFullLight
41 inputVideoFile: MVideoFile
44 const { video, inputVideoFile, job } = options
46 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
47 const newExtname = '.mp4'
49 // Will be released by our transcodeVOD function once ffmpeg is ran
50 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
55 const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
57 const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
58 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
60 const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
64 const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
66 const transcodeOptions: TranscodeVODOptions = {
69 inputPath: videoInputPath,
70 outputPath: videoTranscodedPath,
72 inputFileMutexReleaser,
74 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
75 profile: CONFIG.TRANSCODING.PROFILE,
82 // Could be very long!
83 await transcodeVOD(transcodeOptions)
85 // Important to do this before getVideoFilename() to take in account the new filename
86 inputVideoFile.resolution = resolution
87 inputVideoFile.extname = newExtname
88 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
89 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
91 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
92 await remove(videoInputPath)
94 return { transcodeType, videoFile }
99 inputFileMutexReleaser()
103 // Transcode the original video file to a lower resolution compatible with WebTorrent
104 async function transcodeNewWebTorrentResolution (options: {
105 video: MVideoFullLight
106 resolution: VideoResolution
109 const { video, resolution, job } = options
111 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
112 const newExtname = '.mp4'
114 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
119 const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
121 const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
122 const newVideoFile = new VideoFileModel({
125 filename: generateWebTorrentVideoFilename(resolution, newExtname),
130 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
132 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
134 type: 'only-audio' as 'only-audio',
136 inputPath: videoInputPath,
137 outputPath: videoTranscodedPath,
139 inputFileMutexReleaser,
141 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
142 profile: CONFIG.TRANSCODING.PROFILE,
149 type: 'video' as 'video',
150 inputPath: videoInputPath,
151 outputPath: videoTranscodedPath,
153 inputFileMutexReleaser,
155 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
156 profile: CONFIG.TRANSCODING.PROFILE,
163 await transcodeVOD(transcodeOptions)
165 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
170 inputFileMutexReleaser()
174 // Merge an image with an audio file to create a video
175 async function mergeAudioVideofile (options: {
176 video: MVideoFullLight
177 resolution: VideoResolution
180 const { video, resolution, job } = options
182 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
183 const newExtname = '.mp4'
185 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
190 const inputVideoFile = video.getMinQualityFile()
192 const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
194 const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
195 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
197 // If the user updates the video preview during transcoding
198 const previewPath = video.getPreview().getPath()
199 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
200 await copyFile(previewPath, tmpPreviewPath)
202 const transcodeOptions = {
203 type: 'merge-audio' as 'merge-audio',
205 inputPath: tmpPreviewPath,
206 outputPath: videoTranscodedPath,
208 inputFileMutexReleaser,
210 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
211 profile: CONFIG.TRANSCODING.PROFILE,
213 audioPath: audioInputPath,
220 await transcodeVOD(transcodeOptions)
222 await remove(audioInputPath)
223 await remove(tmpPreviewPath)
225 await remove(tmpPreviewPath)
229 // Important to do this before getVideoFilename() to take in account the new file extension
230 inputVideoFile.extname = newExtname
231 inputVideoFile.resolution = resolution
232 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
234 // ffmpeg generated a new video file, so update the video duration
235 // See https://trac.ffmpeg.org/ticket/5456
236 video.duration = await getVideoStreamDuration(videoTranscodedPath)
239 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
244 inputFileMutexReleaser()
248 // Concat TS segments from a live video to a fragmented mp4 HLS playlist
249 async function generateHlsPlaylistResolutionFromTS (options: {
251 concatenatedTsFilePath: string
252 resolution: VideoResolution
254 inputFileMutexReleaser: MutexInterface.Releaser
256 return generateHlsPlaylistCommon({
257 type: 'hls-from-ts' as 'hls-from-ts',
258 inputPath: options.concatenatedTsFilePath,
260 ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
264 // Generate an HLS playlist from an input file, and update the master playlist
265 function generateHlsPlaylistResolution (options: {
267 videoInputPath: string
268 resolution: VideoResolution
270 inputFileMutexReleaser: MutexInterface.Releaser
273 return generateHlsPlaylistCommon({
274 type: 'hls' as 'hls',
275 inputPath: options.videoInputPath,
277 ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
281 // ---------------------------------------------------------------------------
284 generateHlsPlaylistResolution,
285 generateHlsPlaylistResolutionFromTS,
286 optimizeOriginalVideofile,
287 transcodeNewWebTorrentResolution,
291 // ---------------------------------------------------------------------------
293 async function onWebTorrentVideoFileTranscoding (
294 video: MVideoFullLight,
295 videoFile: MVideoFile,
296 transcodingPath: string,
297 newVideoFile: MVideoFile
299 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
304 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
306 const stats = await stat(transcodingPath)
308 const probe = await ffprobePromise(transcodingPath)
309 const fps = await getVideoStreamFPS(transcodingPath, probe)
310 const metadata = await buildFileMetadata(transcodingPath, probe)
312 await move(transcodingPath, outputPath, { overwrite: true })
314 videoFile.size = stats.size
316 videoFile.metadata = metadata
318 await createTorrentAndSetInfoHash(video, videoFile)
320 const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
321 if (oldFile) await video.removeWebTorrentFile(oldFile)
323 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
324 video.VideoFiles = await video.$get('VideoFiles')
326 return { video, videoFile }
332 async function generateHlsPlaylistCommon (options: {
333 type: 'hls' | 'hls-from-ts'
336 resolution: VideoResolution
338 inputFileMutexReleaser: MutexInterface.Releaser
345 const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
346 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
348 const videoTranscodedBasePath = join(transcodeDirectory, type)
349 await ensureDir(videoTranscodedBasePath)
351 const videoFilename = generateHLSVideoFilename(resolution)
352 const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
353 const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
355 const transcodeOptions = {
359 outputPath: resolutionPlaylistFileTranscodePath,
361 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
362 profile: CONFIG.TRANSCODING.PROFILE,
369 inputFileMutexReleaser,
378 await transcodeVOD(transcodeOptions)
380 // Create or update the playlist
381 const playlist = await retryTransactionWrapper(() => {
382 return sequelizeTypescript.transaction(async transaction => {
383 return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
387 const newVideoFile = new VideoFileModel({
389 extname: extnameUtil(videoFilename),
391 filename: videoFilename,
393 videoStreamingPlaylistId: playlist.id
396 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
399 // VOD transcoding is a long task, refresh video attributes
402 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
403 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
405 // Move playlist file
406 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
407 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
409 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
411 // Update video duration if it was not set (in case of a live for example)
412 if (!video.duration) {
413 video.duration = await getVideoStreamDuration(videoFilePath)
417 const stats = await stat(videoFilePath)
419 newVideoFile.size = stats.size
420 newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
421 newVideoFile.metadata = await buildFileMetadata(videoFilePath)
423 await createTorrentAndSetInfoHash(playlist, newVideoFile)
425 const oldFile = await VideoFileModel.loadHLSFile({
426 playlistId: playlist.id,
427 fps: newVideoFile.fps,
428 resolution: newVideoFile.resolution
432 await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
433 await oldFile.destroy()
436 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
438 await updatePlaylistAfterFileChange(video, playlist)
440 return { resolutionPlaylistPath, videoFile: savedVideoFile }
446 function buildOriginalFileResolution (inputResolution: number) {
447 if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) return toEven(inputResolution)
449 const resolutions = computeResolutionsToTranscode({
450 input: inputResolution,
454 // We don't really care about the audio resolution in this context
458 if (resolutions.length === 0) return toEven(inputResolution)
460 return Math.max(...resolutions)