]>
Commit | Line | Data |
---|---|---|
1 | import { MutexInterface } from 'async-mutex' | |
2 | import { Job } from 'bullmq' | |
3 | import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' | |
4 | import { basename, extname as extnameUtil, join } from 'path' | |
5 | import { toEven } from '@server/helpers/core-utils' | |
6 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | |
7 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | |
8 | import { sequelizeTypescript } from '@server/initializers/database' | |
9 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | |
10 | import { pick } from '@shared/core-utils' | |
11 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' | |
12 | import { | |
13 | buildFileMetadata, | |
14 | canDoQuickTranscode, | |
15 | computeResolutionsToTranscode, | |
16 | ffprobePromise, | |
17 | getVideoStreamDuration, | |
18 | getVideoStreamFPS, | |
19 | transcodeVOD, | |
20 | TranscodeVODOptions, | |
21 | TranscodeVODOptionsType | |
22 | } from '../../helpers/ffmpeg' | |
23 | import { CONFIG } from '../../initializers/config' | |
24 | import { VideoFileModel } from '../../models/video/video-file' | |
25 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | |
26 | import { updatePlaylistAfterFileChange } from '../hls' | |
27 | import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' | |
28 | import { VideoPathManager } from '../video-path-manager' | |
29 | import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' | |
30 | ||
31 | /** | |
32 | * | |
33 | * Functions that run transcoding functions, update the database, cleanup files, create torrent files... | |
34 | * Mainly called by the job queue | |
35 | * | |
36 | */ | |
37 | ||
38 | // Optimize the original video file and replace it. The resolution is not changed. | |
39 | async function optimizeOriginalVideofile (options: { | |
40 | video: MVideoFullLight | |
41 | inputVideoFile: MVideoFile | |
42 | job: Job | |
43 | }) { | |
44 | const { video, inputVideoFile, job } = options | |
45 | ||
46 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | |
47 | const newExtname = '.mp4' | |
48 | ||
49 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | |
50 | ||
51 | try { | |
52 | await video.reload() | |
53 | ||
54 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) | |
55 | ||
56 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { | |
57 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | |
58 | ||
59 | const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) | |
60 | ? 'quick-transcode' | |
61 | : 'video' | |
62 | ||
63 | const resolution = buildOriginalFileResolution(inputVideoFile.resolution) | |
64 | ||
65 | const transcodeOptions: TranscodeVODOptions = { | |
66 | type: transcodeType, | |
67 | ||
68 | inputPath: videoInputPath, | |
69 | outputPath: videoTranscodedPath, | |
70 | ||
71 | inputFileMutexReleaser, | |
72 | ||
73 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | |
74 | profile: CONFIG.TRANSCODING.PROFILE, | |
75 | ||
76 | resolution, | |
77 | ||
78 | job | |
79 | } | |
80 | ||
81 | // Could be very long! | |
82 | await transcodeVOD(transcodeOptions) | |
83 | ||
84 | // Important to do this before getVideoFilename() to take in account the new filename | |
85 | inputVideoFile.resolution = resolution | |
86 | inputVideoFile.extname = newExtname | |
87 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | |
88 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | |
89 | ||
90 | const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) | |
91 | await remove(videoInputPath) | |
92 | ||
93 | return { transcodeType, videoFile } | |
94 | }) | |
95 | ||
96 | return result | |
97 | } finally { | |
98 | inputFileMutexReleaser() | |
99 | } | |
100 | } | |
101 | ||
102 | // Transcode the original video file to a lower resolution compatible with WebTorrent | |
103 | async function transcodeNewWebTorrentResolution (options: { | |
104 | video: MVideoFullLight | |
105 | resolution: VideoResolution | |
106 | job: Job | |
107 | }) { | |
108 | const { video, resolution, job } = options | |
109 | ||
110 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | |
111 | const newExtname = '.mp4' | |
112 | ||
113 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | |
114 | ||
115 | try { | |
116 | await video.reload() | |
117 | ||
118 | const file = video.getMaxQualityFile().withVideoOrPlaylist(video) | |
119 | ||
120 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { | |
121 | const newVideoFile = new VideoFileModel({ | |
122 | resolution, | |
123 | extname: newExtname, | |
124 | filename: generateWebTorrentVideoFilename(resolution, newExtname), | |
125 | size: 0, | |
126 | videoId: video.id | |
127 | }) | |
128 | ||
129 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) | |
130 | ||
131 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO | |
132 | ? { | |
133 | type: 'only-audio' as 'only-audio', | |
134 | ||
135 | inputPath: videoInputPath, | |
136 | outputPath: videoTranscodedPath, | |
137 | ||
138 | inputFileMutexReleaser, | |
139 | ||
140 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | |
141 | profile: CONFIG.TRANSCODING.PROFILE, | |
142 | ||
143 | resolution, | |
144 | ||
145 | job | |
146 | } | |
147 | : { | |
148 | type: 'video' as 'video', | |
149 | inputPath: videoInputPath, | |
150 | outputPath: videoTranscodedPath, | |
151 | ||
152 | inputFileMutexReleaser, | |
153 | ||
154 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | |
155 | profile: CONFIG.TRANSCODING.PROFILE, | |
156 | ||
157 | resolution, | |
158 | ||
159 | job | |
160 | } | |
161 | ||
162 | await transcodeVOD(transcodeOptions) | |
163 | ||
164 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile) | |
165 | }) | |
166 | ||
167 | return result | |
168 | } finally { | |
169 | inputFileMutexReleaser() | |
170 | } | |
171 | } | |
172 | ||
173 | // Merge an image with an audio file to create a video | |
174 | async function mergeAudioVideofile (options: { | |
175 | video: MVideoFullLight | |
176 | resolution: VideoResolution | |
177 | job: Job | |
178 | }) { | |
179 | const { video, resolution, job } = options | |
180 | ||
181 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | |
182 | const newExtname = '.mp4' | |
183 | ||
184 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | |
185 | ||
186 | try { | |
187 | await video.reload() | |
188 | ||
189 | const inputVideoFile = video.getMinQualityFile() | |
190 | ||
191 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) | |
192 | ||
193 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { | |
194 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | |
195 | ||
196 | // If the user updates the video preview during transcoding | |
197 | const previewPath = video.getPreview().getPath() | |
198 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | |
199 | await copyFile(previewPath, tmpPreviewPath) | |
200 | ||
201 | const transcodeOptions = { | |
202 | type: 'merge-audio' as 'merge-audio', | |
203 | ||
204 | inputPath: tmpPreviewPath, | |
205 | outputPath: videoTranscodedPath, | |
206 | ||
207 | inputFileMutexReleaser, | |
208 | ||
209 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | |
210 | profile: CONFIG.TRANSCODING.PROFILE, | |
211 | ||
212 | audioPath: audioInputPath, | |
213 | resolution, | |
214 | ||
215 | job | |
216 | } | |
217 | ||
218 | try { | |
219 | await transcodeVOD(transcodeOptions) | |
220 | ||
221 | await remove(audioInputPath) | |
222 | await remove(tmpPreviewPath) | |
223 | } catch (err) { | |
224 | await remove(tmpPreviewPath) | |
225 | throw err | |
226 | } | |
227 | ||
228 | // Important to do this before getVideoFilename() to take in account the new file extension | |
229 | inputVideoFile.extname = newExtname | |
230 | inputVideoFile.resolution = resolution | |
231 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | |
232 | ||
233 | // ffmpeg generated a new video file, so update the video duration | |
234 | // See https://trac.ffmpeg.org/ticket/5456 | |
235 | video.duration = await getVideoStreamDuration(videoTranscodedPath) | |
236 | await video.save() | |
237 | ||
238 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) | |
239 | }) | |
240 | ||
241 | return result | |
242 | } finally { | |
243 | inputFileMutexReleaser() | |
244 | } | |
245 | } | |
246 | ||
247 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist | |
248 | async function generateHlsPlaylistResolutionFromTS (options: { | |
249 | video: MVideo | |
250 | concatenatedTsFilePath: string | |
251 | resolution: VideoResolution | |
252 | isAAC: boolean | |
253 | inputFileMutexReleaser: MutexInterface.Releaser | |
254 | }) { | |
255 | return generateHlsPlaylistCommon({ | |
256 | type: 'hls-from-ts' as 'hls-from-ts', | |
257 | inputPath: options.concatenatedTsFilePath, | |
258 | ||
259 | ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ]) | |
260 | }) | |
261 | } | |
262 | ||
263 | // Generate an HLS playlist from an input file, and update the master playlist | |
264 | function generateHlsPlaylistResolution (options: { | |
265 | video: MVideo | |
266 | videoInputPath: string | |
267 | resolution: VideoResolution | |
268 | copyCodecs: boolean | |
269 | inputFileMutexReleaser: MutexInterface.Releaser | |
270 | job?: Job | |
271 | }) { | |
272 | return generateHlsPlaylistCommon({ | |
273 | type: 'hls' as 'hls', | |
274 | inputPath: options.videoInputPath, | |
275 | ||
276 | ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) | |
277 | }) | |
278 | } | |
279 | ||
280 | // --------------------------------------------------------------------------- | |
281 | ||
282 | export { | |
283 | generateHlsPlaylistResolution, | |
284 | generateHlsPlaylistResolutionFromTS, | |
285 | optimizeOriginalVideofile, | |
286 | transcodeNewWebTorrentResolution, | |
287 | mergeAudioVideofile | |
288 | } | |
289 | ||
290 | // --------------------------------------------------------------------------- | |
291 | ||
292 | async function onWebTorrentVideoFileTranscoding ( | |
293 | video: MVideoFullLight, | |
294 | videoFile: MVideoFile, | |
295 | transcodingPath: string, | |
296 | newVideoFile: MVideoFile | |
297 | ) { | |
298 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | |
299 | ||
300 | try { | |
301 | await video.reload() | |
302 | ||
303 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) | |
304 | ||
305 | const stats = await stat(transcodingPath) | |
306 | ||
307 | const probe = await ffprobePromise(transcodingPath) | |
308 | const fps = await getVideoStreamFPS(transcodingPath, probe) | |
309 | const metadata = await buildFileMetadata(transcodingPath, probe) | |
310 | ||
311 | await move(transcodingPath, outputPath, { overwrite: true }) | |
312 | ||
313 | videoFile.size = stats.size | |
314 | videoFile.fps = fps | |
315 | videoFile.metadata = metadata | |
316 | ||
317 | await createTorrentAndSetInfoHash(video, videoFile) | |
318 | ||
319 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) | |
320 | if (oldFile) await video.removeWebTorrentFile(oldFile) | |
321 | ||
322 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | |
323 | video.VideoFiles = await video.$get('VideoFiles') | |
324 | ||
325 | return { video, videoFile } | |
326 | } finally { | |
327 | mutexReleaser() | |
328 | } | |
329 | } | |
330 | ||
331 | async function generateHlsPlaylistCommon (options: { | |
332 | type: 'hls' | 'hls-from-ts' | |
333 | video: MVideo | |
334 | inputPath: string | |
335 | resolution: VideoResolution | |
336 | ||
337 | inputFileMutexReleaser: MutexInterface.Releaser | |
338 | ||
339 | copyCodecs?: boolean | |
340 | isAAC?: boolean | |
341 | ||
342 | job?: Job | |
343 | }) { | |
344 | const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options | |
345 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | |
346 | ||
347 | const videoTranscodedBasePath = join(transcodeDirectory, type) | |
348 | await ensureDir(videoTranscodedBasePath) | |
349 | ||
350 | const videoFilename = generateHLSVideoFilename(resolution) | |
351 | const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) | |
352 | const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename) | |
353 | ||
354 | const transcodeOptions = { | |
355 | type, | |
356 | ||
357 | inputPath, | |
358 | outputPath: resolutionPlaylistFileTranscodePath, | |
359 | ||
360 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | |
361 | profile: CONFIG.TRANSCODING.PROFILE, | |
362 | ||
363 | resolution, | |
364 | copyCodecs, | |
365 | ||
366 | isAAC, | |
367 | ||
368 | inputFileMutexReleaser, | |
369 | ||
370 | hlsPlaylist: { | |
371 | videoFilename | |
372 | }, | |
373 | ||
374 | job | |
375 | } | |
376 | ||
377 | await transcodeVOD(transcodeOptions) | |
378 | ||
379 | // Create or update the playlist | |
380 | const playlist = await retryTransactionWrapper(() => { | |
381 | return sequelizeTypescript.transaction(async transaction => { | |
382 | return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) | |
383 | }) | |
384 | }) | |
385 | ||
386 | const newVideoFile = new VideoFileModel({ | |
387 | resolution, | |
388 | extname: extnameUtil(videoFilename), | |
389 | size: 0, | |
390 | filename: videoFilename, | |
391 | fps: -1, | |
392 | videoStreamingPlaylistId: playlist.id | |
393 | }) | |
394 | ||
395 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | |
396 | ||
397 | try { | |
398 | // VOD transcoding is a long task, refresh video attributes | |
399 | await video.reload() | |
400 | ||
401 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) | |
402 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) | |
403 | ||
404 | // Move playlist file | |
405 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) | |
406 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) | |
407 | // Move video file | |
408 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) | |
409 | ||
410 | // Update video duration if it was not set (in case of a live for example) | |
411 | if (!video.duration) { | |
412 | video.duration = await getVideoStreamDuration(videoFilePath) | |
413 | await video.save() | |
414 | } | |
415 | ||
416 | const stats = await stat(videoFilePath) | |
417 | ||
418 | newVideoFile.size = stats.size | |
419 | newVideoFile.fps = await getVideoStreamFPS(videoFilePath) | |
420 | newVideoFile.metadata = await buildFileMetadata(videoFilePath) | |
421 | ||
422 | await createTorrentAndSetInfoHash(playlist, newVideoFile) | |
423 | ||
424 | const oldFile = await VideoFileModel.loadHLSFile({ | |
425 | playlistId: playlist.id, | |
426 | fps: newVideoFile.fps, | |
427 | resolution: newVideoFile.resolution | |
428 | }) | |
429 | ||
430 | if (oldFile) { | |
431 | await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | |
432 | await oldFile.destroy() | |
433 | } | |
434 | ||
435 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) | |
436 | ||
437 | await updatePlaylistAfterFileChange(video, playlist) | |
438 | ||
439 | return { resolutionPlaylistPath, videoFile: savedVideoFile } | |
440 | } finally { | |
441 | mutexReleaser() | |
442 | } | |
443 | } | |
444 | ||
445 | function buildOriginalFileResolution (inputResolution: number) { | |
446 | if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) return toEven(inputResolution) | |
447 | ||
448 | const resolutions = computeResolutionsToTranscode({ input: inputResolution, type: 'vod', includeInput: false, strictLower: false }) | |
449 | if (resolutions.length === 0) return toEven(inputResolution) | |
450 | ||
451 | return Math.max(...resolutions) | |
452 | } |