]>
Commit | Line | Data |
---|---|---|
3545e72c | 1 | import { MutexInterface } from 'async-mutex' |
5a921e7b | 2 | import { Job } from 'bullmq' |
3851e732 | 3 | import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' |
d7a25329 | 4 | import { basename, extname as extnameUtil, join } from 'path' |
318b0bd0 | 5 | import { toEven } from '@server/helpers/core-utils' |
7b6b445d | 6 | import { retryTransactionWrapper } from '@server/helpers/database-utils' |
053aed43 | 7 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
7b6b445d | 8 | import { sequelizeTypescript } from '@server/initializers/database' |
1bb4c9ab | 9 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
3545e72c | 10 | import { pick } from '@shared/core-utils' |
0305db28 | 11 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' |
c729caf6 | 12 | import { |
4ec52d04 | 13 | buildFileMetadata, |
c729caf6 | 14 | canDoQuickTranscode, |
84cae54e | 15 | computeResolutionsToTranscode, |
3545e72c | 16 | ffprobePromise, |
c729caf6 | 17 | getVideoStreamDuration, |
c729caf6 C |
18 | getVideoStreamFPS, |
19 | transcodeVOD, | |
20 | TranscodeVODOptions, | |
21 | TranscodeVODOptionsType | |
22 | } from '../../helpers/ffmpeg' | |
c07902b9 | 23 | import { CONFIG } from '../../initializers/config' |
c07902b9 C |
24 | import { VideoFileModel } from '../../models/video/video-file' |
25 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | |
1bb4c9ab C |
26 | import { updatePlaylistAfterFileChange } from '../hls' |
27 | import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' | |
0305db28 | 28 | import { VideoPathManager } from '../video-path-manager' |
c729caf6 | 29 | import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' |
098eb377 | 30 | |
658a47ab | 31 | /** |
6b67897e C |
32 | * |
33 | * Functions that run transcoding functions, update the database, cleanup files, create torrent files... | |
34 | * Mainly called by the job queue | |
35 | * | |
658a47ab | 36 | */ |
6b67897e C |
37 | |
38 | // Optimize the original video file and replace it. The resolution is not changed. | |
3545e72c | 39 | async function optimizeOriginalVideofile (options: { |
84cae54e C |
40 | video: MVideoFullLight |
41 | inputVideoFile: MVideoFile | |
42 | job: Job | |
43 | }) { | |
44 | const { video, inputVideoFile, job } = options | |
45 | ||
2fbd5e25 | 46 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
098eb377 | 47 | const newExtname = '.mp4' |
9f1ddd24 | 48 | |
6740b642 | 49 | // Will be released by our transcodeVOD function once ffmpeg is ran |
3545e72c | 50 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
098eb377 | 51 | |
3545e72c C |
52 | try { |
53 | await video.reload() | |
5ba49f26 | 54 | |
3545e72c | 55 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) |
318b0bd0 | 56 | |
3545e72c C |
57 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { |
58 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | |
9252a33d | 59 | |
3545e72c C |
60 | const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) |
61 | ? 'quick-transcode' | |
62 | : 'video' | |
9252a33d | 63 | |
3545e72c | 64 | const resolution = buildOriginalFileResolution(inputVideoFile.resolution) |
9252a33d | 65 | |
3545e72c C |
66 | const transcodeOptions: TranscodeVODOptions = { |
67 | type: transcodeType, | |
3b01f4c0 | 68 | |
3545e72c C |
69 | inputPath: videoInputPath, |
70 | outputPath: videoTranscodedPath, | |
098eb377 | 71 | |
3545e72c | 72 | inputFileMutexReleaser, |
098eb377 | 73 | |
3545e72c C |
74 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
75 | profile: CONFIG.TRANSCODING.PROFILE, | |
098eb377 | 76 | |
3545e72c | 77 | resolution, |
2fbd5e25 | 78 | |
3545e72c C |
79 | job |
80 | } | |
098eb377 | 81 | |
3545e72c C |
82 | // Could be very long! |
83 | await transcodeVOD(transcodeOptions) | |
84 | ||
85 | // Important to do this before getVideoFilename() to take in account the new filename | |
86 | inputVideoFile.resolution = resolution | |
87 | inputVideoFile.extname = newExtname | |
88 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | |
89 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | |
90 | ||
91 | const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) | |
92 | await remove(videoInputPath) | |
93 | ||
94 | return { transcodeType, videoFile } | |
95 | }) | |
96 | ||
97 | return result | |
98 | } finally { | |
99 | inputFileMutexReleaser() | |
100 | } | |
098eb377 C |
101 | } |
102 | ||
84cae54e | 103 | // Transcode the original video file to a lower resolution compatible with WebTorrent |
3545e72c | 104 | async function transcodeNewWebTorrentResolution (options: { |
84cae54e C |
105 | video: MVideoFullLight |
106 | resolution: VideoResolution | |
107 | job: Job | |
108 | }) { | |
109 | const { video, resolution, job } = options | |
110 | ||
2fbd5e25 | 111 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
84cae54e | 112 | const newExtname = '.mp4' |
098eb377 | 113 | |
3545e72c | 114 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
098eb377 | 115 | |
3545e72c C |
116 | try { |
117 | await video.reload() | |
90a8bd30 | 118 | |
3545e72c | 119 | const file = video.getMaxQualityFile().withVideoOrPlaylist(video) |
098eb377 | 120 | |
3545e72c C |
121 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { |
122 | const newVideoFile = new VideoFileModel({ | |
123 | resolution, | |
124 | extname: newExtname, | |
125 | filename: generateWebTorrentVideoFilename(resolution, newExtname), | |
126 | size: 0, | |
127 | videoId: video.id | |
128 | }) | |
9252a33d | 129 | |
3545e72c | 130 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) |
9252a33d | 131 | |
3545e72c C |
132 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO |
133 | ? { | |
134 | type: 'only-audio' as 'only-audio', | |
9252a33d | 135 | |
3545e72c C |
136 | inputPath: videoInputPath, |
137 | outputPath: videoTranscodedPath, | |
3b01f4c0 | 138 | |
3545e72c | 139 | inputFileMutexReleaser, |
9252a33d | 140 | |
3545e72c C |
141 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
142 | profile: CONFIG.TRANSCODING.PROFILE, | |
9252a33d | 143 | |
3545e72c | 144 | resolution, |
3b01f4c0 | 145 | |
3545e72c C |
146 | job |
147 | } | |
148 | : { | |
149 | type: 'video' as 'video', | |
150 | inputPath: videoInputPath, | |
151 | outputPath: videoTranscodedPath, | |
098eb377 | 152 | |
3545e72c C |
153 | inputFileMutexReleaser, |
154 | ||
155 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | |
156 | profile: CONFIG.TRANSCODING.PROFILE, | |
157 | ||
158 | resolution, | |
159 | ||
160 | job | |
161 | } | |
162 | ||
163 | await transcodeVOD(transcodeOptions) | |
164 | ||
165 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile) | |
166 | }) | |
167 | ||
168 | return result | |
169 | } finally { | |
170 | inputFileMutexReleaser() | |
171 | } | |
536598cf C |
172 | } |
173 | ||
6b67897e | 174 | // Merge an image with an audio file to create a video |
3545e72c | 175 | async function mergeAudioVideofile (options: { |
84cae54e C |
176 | video: MVideoFullLight |
177 | resolution: VideoResolution | |
178 | job: Job | |
179 | }) { | |
180 | const { video, resolution, job } = options | |
181 | ||
536598cf C |
182 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
183 | const newExtname = '.mp4' | |
184 | ||
3545e72c | 185 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
2fbd5e25 | 186 | |
3545e72c C |
187 | try { |
188 | await video.reload() | |
098eb377 | 189 | |
3545e72c | 190 | const inputVideoFile = video.getMinQualityFile() |
eba06469 | 191 | |
3545e72c | 192 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) |
9252a33d | 193 | |
3545e72c C |
194 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { |
195 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | |
9252a33d | 196 | |
3545e72c C |
197 | // If the user updates the video preview during transcoding |
198 | const previewPath = video.getPreview().getPath() | |
199 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | |
200 | await copyFile(previewPath, tmpPreviewPath) | |
9252a33d | 201 | |
3545e72c C |
202 | const transcodeOptions = { |
203 | type: 'merge-audio' as 'merge-audio', | |
3b01f4c0 | 204 | |
3545e72c C |
205 | inputPath: tmpPreviewPath, |
206 | outputPath: videoTranscodedPath, | |
098eb377 | 207 | |
3545e72c | 208 | inputFileMutexReleaser, |
098eb377 | 209 | |
3545e72c C |
210 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
211 | profile: CONFIG.TRANSCODING.PROFILE, | |
098eb377 | 212 | |
3545e72c C |
213 | audioPath: audioInputPath, |
214 | resolution, | |
536598cf | 215 | |
3545e72c C |
216 | job |
217 | } | |
536598cf | 218 | |
3545e72c C |
219 | try { |
220 | await transcodeVOD(transcodeOptions) | |
221 | ||
222 | await remove(audioInputPath) | |
223 | await remove(tmpPreviewPath) | |
224 | } catch (err) { | |
225 | await remove(tmpPreviewPath) | |
226 | throw err | |
227 | } | |
228 | ||
229 | // Important to do this before getVideoFilename() to take in account the new file extension | |
230 | inputVideoFile.extname = newExtname | |
231 | inputVideoFile.resolution = resolution | |
232 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | |
233 | ||
234 | // ffmpeg generated a new video file, so update the video duration | |
235 | // See https://trac.ffmpeg.org/ticket/5456 | |
236 | video.duration = await getVideoStreamDuration(videoTranscodedPath) | |
237 | await video.save() | |
238 | ||
239 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) | |
240 | }) | |
241 | ||
242 | return result | |
243 | } finally { | |
244 | inputFileMutexReleaser() | |
245 | } | |
098eb377 C |
246 | } |
247 | ||
2650d6d4 | 248 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist |
24516aa2 | 249 | async function generateHlsPlaylistResolutionFromTS (options: { |
4ec52d04 | 250 | video: MVideo |
3851e732 | 251 | concatenatedTsFilePath: string |
2650d6d4 | 252 | resolution: VideoResolution |
e772bdf1 | 253 | isAAC: boolean |
3545e72c | 254 | inputFileMutexReleaser: MutexInterface.Releaser |
2650d6d4 | 255 | }) { |
3851e732 | 256 | return generateHlsPlaylistCommon({ |
e772bdf1 | 257 | type: 'hls-from-ts' as 'hls-from-ts', |
3545e72c C |
258 | inputPath: options.concatenatedTsFilePath, |
259 | ||
260 | ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ]) | |
3851e732 | 261 | }) |
2650d6d4 C |
262 | } |
263 | ||
6b67897e | 264 | // Generate an HLS playlist from an input file, and update the master playlist |
24516aa2 | 265 | function generateHlsPlaylistResolution (options: { |
4ec52d04 | 266 | video: MVideo |
b5b68755 C |
267 | videoInputPath: string |
268 | resolution: VideoResolution | |
269 | copyCodecs: boolean | |
3545e72c | 270 | inputFileMutexReleaser: MutexInterface.Releaser |
3b01f4c0 | 271 | job?: Job |
b5b68755 | 272 | }) { |
2650d6d4 | 273 | return generateHlsPlaylistCommon({ |
3b01f4c0 | 274 | type: 'hls' as 'hls', |
3545e72c C |
275 | inputPath: options.videoInputPath, |
276 | ||
277 | ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) | |
2650d6d4 C |
278 | }) |
279 | } | |
280 | ||
281 | // --------------------------------------------------------------------------- | |
282 | ||
283 | export { | |
24516aa2 C |
284 | generateHlsPlaylistResolution, |
285 | generateHlsPlaylistResolutionFromTS, | |
2650d6d4 | 286 | optimizeOriginalVideofile, |
24516aa2 | 287 | transcodeNewWebTorrentResolution, |
1bcb03a1 | 288 | mergeAudioVideofile |
2650d6d4 C |
289 | } |
290 | ||
291 | // --------------------------------------------------------------------------- | |
292 | ||
24516aa2 | 293 | async function onWebTorrentVideoFileTranscoding ( |
90a8bd30 | 294 | video: MVideoFullLight, |
24516aa2 C |
295 | videoFile: MVideoFile, |
296 | transcodingPath: string, | |
3545e72c | 297 | newVideoFile: MVideoFile |
24516aa2 | 298 | ) { |
3545e72c C |
299 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
300 | ||
301 | try { | |
302 | await video.reload() | |
303 | ||
304 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) | |
2650d6d4 | 305 | |
3545e72c | 306 | const stats = await stat(transcodingPath) |
2650d6d4 | 307 | |
3545e72c C |
308 | const probe = await ffprobePromise(transcodingPath) |
309 | const fps = await getVideoStreamFPS(transcodingPath, probe) | |
310 | const metadata = await buildFileMetadata(transcodingPath, probe) | |
2650d6d4 | 311 | |
3545e72c | 312 | await move(transcodingPath, outputPath, { overwrite: true }) |
2650d6d4 | 313 | |
3545e72c C |
314 | videoFile.size = stats.size |
315 | videoFile.fps = fps | |
316 | videoFile.metadata = metadata | |
7b6b445d | 317 | |
3545e72c | 318 | await createTorrentAndSetInfoHash(video, videoFile) |
2650d6d4 | 319 | |
3545e72c C |
320 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) |
321 | if (oldFile) await video.removeWebTorrentFile(oldFile) | |
322 | ||
323 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | |
324 | video.VideoFiles = await video.$get('VideoFiles') | |
325 | ||
326 | return { video, videoFile } | |
327 | } finally { | |
328 | mutexReleaser() | |
329 | } | |
2650d6d4 C |
330 | } |
331 | ||
332 | async function generateHlsPlaylistCommon (options: { | |
333 | type: 'hls' | 'hls-from-ts' | |
4ec52d04 | 334 | video: MVideo |
2650d6d4 C |
335 | inputPath: string |
336 | resolution: VideoResolution | |
3545e72c C |
337 | |
338 | inputFileMutexReleaser: MutexInterface.Releaser | |
339 | ||
2650d6d4 | 340 | copyCodecs?: boolean |
e772bdf1 | 341 | isAAC?: boolean |
3b01f4c0 C |
342 | |
343 | job?: Job | |
2650d6d4 | 344 | }) { |
3545e72c | 345 | const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options |
aaedadd5 | 346 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
b5b68755 | 347 | |
9129b769 | 348 | const videoTranscodedBasePath = join(transcodeDirectory, type) |
aaedadd5 | 349 | await ensureDir(videoTranscodedBasePath) |
09209296 | 350 | |
83903cb6 | 351 | const videoFilename = generateHLSVideoFilename(resolution) |
764b1a14 C |
352 | const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) |
353 | const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename) | |
09209296 C |
354 | |
355 | const transcodeOptions = { | |
2650d6d4 | 356 | type, |
9252a33d | 357 | |
2650d6d4 | 358 | inputPath, |
764b1a14 | 359 | outputPath: resolutionPlaylistFileTranscodePath, |
9252a33d | 360 | |
529b3752 | 361 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
1896bca0 | 362 | profile: CONFIG.TRANSCODING.PROFILE, |
9252a33d | 363 | |
09209296 | 364 | resolution, |
d7a25329 | 365 | copyCodecs, |
4c280004 | 366 | |
e772bdf1 C |
367 | isAAC, |
368 | ||
3545e72c C |
369 | inputFileMutexReleaser, |
370 | ||
4c280004 | 371 | hlsPlaylist: { |
d7a25329 | 372 | videoFilename |
3b01f4c0 C |
373 | }, |
374 | ||
375 | job | |
09209296 C |
376 | } |
377 | ||
c729caf6 | 378 | await transcodeVOD(transcodeOptions) |
09209296 | 379 | |
aaedadd5 | 380 | // Create or update the playlist |
1bb4c9ab | 381 | const playlist = await retryTransactionWrapper(() => { |
3b052510 | 382 | return sequelizeTypescript.transaction(async transaction => { |
1bb4c9ab | 383 | return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) |
3b052510 C |
384 | }) |
385 | }) | |
d7a25329 C |
386 | |
387 | const newVideoFile = new VideoFileModel({ | |
388 | resolution, | |
1bb4c9ab | 389 | extname: extnameUtil(videoFilename), |
d7a25329 | 390 | size: 0, |
83903cb6 | 391 | filename: videoFilename, |
d7a25329 | 392 | fps: -1, |
764b1a14 | 393 | videoStreamingPlaylistId: playlist.id |
09209296 | 394 | }) |
d7a25329 | 395 | |
3545e72c | 396 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
aaedadd5 | 397 | |
3545e72c C |
398 | try { |
399 | // VOD transcoding is a long task, refresh video attributes | |
400 | await video.reload() | |
aaedadd5 | 401 | |
3545e72c C |
402 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) |
403 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) | |
e4fc3697 | 404 | |
3545e72c C |
405 | // Move playlist file |
406 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) | |
407 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) | |
408 | // Move video file | |
409 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) | |
d7a25329 | 410 | |
3545e72c C |
411 | // Update video duration if it was not set (in case of a live for example) |
412 | if (!video.duration) { | |
413 | video.duration = await getVideoStreamDuration(videoFilePath) | |
414 | await video.save() | |
415 | } | |
d7a25329 | 416 | |
3545e72c | 417 | const stats = await stat(videoFilePath) |
d7a25329 | 418 | |
3545e72c C |
419 | newVideoFile.size = stats.size |
420 | newVideoFile.fps = await getVideoStreamFPS(videoFilePath) | |
421 | newVideoFile.metadata = await buildFileMetadata(videoFilePath) | |
422 | ||
423 | await createTorrentAndSetInfoHash(playlist, newVideoFile) | |
7b6b445d | 424 | |
3545e72c C |
425 | const oldFile = await VideoFileModel.loadHLSFile({ |
426 | playlistId: playlist.id, | |
427 | fps: newVideoFile.fps, | |
428 | resolution: newVideoFile.resolution | |
429 | }) | |
430 | ||
431 | if (oldFile) { | |
432 | await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | |
433 | await oldFile.destroy() | |
434 | } | |
d7a25329 | 435 | |
3545e72c | 436 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) |
d7a25329 | 437 | |
3545e72c C |
438 | await updatePlaylistAfterFileChange(video, playlist) |
439 | ||
440 | return { resolutionPlaylistPath, videoFile: savedVideoFile } | |
441 | } finally { | |
442 | mutexReleaser() | |
443 | } | |
098eb377 | 444 | } |
84cae54e C |
445 | |
446 | function buildOriginalFileResolution (inputResolution: number) { | |
f30ef8cf C |
447 | if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { |
448 | return toEven(inputResolution) | |
449 | } | |
84cae54e | 450 | |
a32bf8cd C |
451 | const resolutions = computeResolutionsToTranscode({ |
452 | input: inputResolution, | |
453 | type: 'vod', | |
454 | includeInput: false, | |
455 | strictLower: false, | |
456 | // We don't really care about the audio resolution in this context | |
457 | hasAudio: true | |
458 | }) | |
459 | ||
f30ef8cf C |
460 | if (resolutions.length === 0) { |
461 | return toEven(inputResolution) | |
462 | } | |
84cae54e C |
463 | |
464 | return Math.max(...resolutions) | |
465 | } |