-import { Job } from 'bull'
+import { MutexInterface } from 'async-mutex'
+import { Job } from 'bullmq'
import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
import { basename, extname as extnameUtil, join } from 'path'
import { toEven } from '@server/helpers/core-utils'
+import { retryTransactionWrapper } from '@server/helpers/database-utils'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
-import { MStreamingPlaylistFilesVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
+import { pick } from '@shared/core-utils'
import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
-import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import {
buildFileMetadata,
canDoQuickTranscode,
+ computeResolutionsToTranscode,
+ ffprobePromise,
getVideoStreamDuration,
getVideoStreamFPS,
transcodeVOD,
TranscodeVODOptionsType
} from '../../helpers/ffmpeg'
import { CONFIG } from '../../initializers/config'
-import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
import { VideoFileModel } from '../../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
-import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
-import {
- generateHLSMasterPlaylistFilename,
- generateHlsSha256SegmentsFilename,
- generateHLSVideoFilename,
- generateWebTorrentVideoFilename,
- getHlsResolutionPlaylistFilename
-} from '../paths'
+import { updatePlaylistAfterFileChange } from '../hls'
+import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths'
import { VideoPathManager } from '../video-path-manager'
import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
*/
// Optimize the original video file and replace it. The resolution is not changed.
-function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) {
+async function optimizeOriginalVideofile (options: {
+ video: MVideoFullLight
+ inputVideoFile: MVideoFile
+ job: Job
+}) {
+ const { video, inputVideoFile, job } = options
+
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
- return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => {
- const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
+ // Will be released by our transcodeVOD function once ffmpeg is ran
+ const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
- const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
- ? 'quick-transcode'
- : 'video'
+ try {
+ await video.reload()
- const resolution = toEven(inputVideoFile.resolution)
+ const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
- const transcodeOptions: TranscodeVODOptions = {
- type: transcodeType,
+ const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
+ const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
- inputPath: videoInputPath,
- outputPath: videoTranscodedPath,
+ const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
+ ? 'quick-transcode'
+ : 'video'
- availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
- profile: CONFIG.TRANSCODING.PROFILE,
+ const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
- resolution,
+ const transcodeOptions: TranscodeVODOptions = {
+ type: transcodeType,
- job
- }
+ inputPath: videoInputPath,
+ outputPath: videoTranscodedPath,
- // Could be very long!
- await transcodeVOD(transcodeOptions)
+ inputFileMutexReleaser,
- // Important to do this before getVideoFilename() to take in account the new filename
- inputVideoFile.extname = newExtname
- inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
- inputVideoFile.storage = VideoStorage.FILE_SYSTEM
+ availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
+ profile: CONFIG.TRANSCODING.PROFILE,
- const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
+ resolution,
- const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
- await remove(videoInputPath)
+ job
+ }
- return { transcodeType, videoFile }
- })
-}
+ // Could be very long!
+ await transcodeVOD(transcodeOptions)
-// Transcode the original video file to a lower resolution
-// We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
-function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) {
- const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
- const extname = '.mp4'
-
- return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => {
- const newVideoFile = new VideoFileModel({
- resolution,
- extname,
- filename: generateWebTorrentVideoFilename(resolution, extname),
- size: 0,
- videoId: video.id
+ // Important to do this before getVideoFilename() to take in account the new filename
+ inputVideoFile.resolution = resolution
+ inputVideoFile.extname = newExtname
+ inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
+ inputVideoFile.storage = VideoStorage.FILE_SYSTEM
+
+ const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
+ await remove(videoInputPath)
+
+ return { transcodeType, videoFile }
})
- const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
- const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
+ return result
+ } finally {
+ inputFileMutexReleaser()
+ }
+}
- const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
- ? {
- type: 'only-audio' as 'only-audio',
+// Transcode the original video file to a lower resolution compatible with WebTorrent
+async function transcodeNewWebTorrentResolution (options: {
+ video: MVideoFullLight
+ resolution: VideoResolution
+ job: Job
+}) {
+ const { video, resolution, job } = options
- inputPath: videoInputPath,
- outputPath: videoTranscodedPath,
+ const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
+ const newExtname = '.mp4'
- availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
- profile: CONFIG.TRANSCODING.PROFILE,
+ const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+ try {
+ await video.reload()
+
+ const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
+
+ const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
+ const newVideoFile = new VideoFileModel({
resolution,
+ extname: newExtname,
+ filename: generateWebTorrentVideoFilename(resolution, newExtname),
+ size: 0,
+ videoId: video.id
+ })
- job
- }
- : {
- type: 'video' as 'video',
- inputPath: videoInputPath,
- outputPath: videoTranscodedPath,
+ const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
- availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
- profile: CONFIG.TRANSCODING.PROFILE,
+ const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
+ ? {
+ type: 'only-audio' as 'only-audio',
- resolution,
- isPortraitMode: isPortrait,
+ inputPath: videoInputPath,
+ outputPath: videoTranscodedPath,
- job
- }
+ inputFileMutexReleaser,
- await transcodeVOD(transcodeOptions)
+ availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
+ profile: CONFIG.TRANSCODING.PROFILE,
- return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
- })
+ resolution,
+
+ job
+ }
+ : {
+ type: 'video' as 'video',
+ inputPath: videoInputPath,
+ outputPath: videoTranscodedPath,
+
+ inputFileMutexReleaser,
+
+ availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
+ profile: CONFIG.TRANSCODING.PROFILE,
+
+ resolution,
+
+ job
+ }
+
+ await transcodeVOD(transcodeOptions)
+
+ return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
+ })
+
+ return result
+ } finally {
+ inputFileMutexReleaser()
+ }
}
// Merge an image with an audio file to create a video
-function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) {
+async function mergeAudioVideofile (options: {
+ video: MVideoFullLight
+ resolution: VideoResolution
+ job: Job
+}) {
+ const { video, resolution, job } = options
+
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
- const inputVideoFile = video.getMinQualityFile()
+ const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
- return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => {
- const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
+ try {
+ await video.reload()
- // If the user updates the video preview during transcoding
- const previewPath = video.getPreview().getPath()
- const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
- await copyFile(previewPath, tmpPreviewPath)
+ const inputVideoFile = video.getMinQualityFile()
- const transcodeOptions = {
- type: 'merge-audio' as 'merge-audio',
+ const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
- inputPath: tmpPreviewPath,
- outputPath: videoTranscodedPath,
+ const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
+ const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
- availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
- profile: CONFIG.TRANSCODING.PROFILE,
+ // If the user updates the video preview during transcoding
+ const previewPath = video.getPreview().getPath()
+ const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
+ await copyFile(previewPath, tmpPreviewPath)
- audioPath: audioInputPath,
- resolution,
+ const transcodeOptions = {
+ type: 'merge-audio' as 'merge-audio',
- job
- }
+ inputPath: tmpPreviewPath,
+ outputPath: videoTranscodedPath,
- try {
- await transcodeVOD(transcodeOptions)
+ inputFileMutexReleaser,
- await remove(audioInputPath)
- await remove(tmpPreviewPath)
- } catch (err) {
- await remove(tmpPreviewPath)
- throw err
- }
+ availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
+ profile: CONFIG.TRANSCODING.PROFILE,
+
+ audioPath: audioInputPath,
+ resolution,
- // Important to do this before getVideoFilename() to take in account the new file extension
- inputVideoFile.extname = newExtname
- inputVideoFile.resolution = resolution
- inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
+ job
+ }
- const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
- // ffmpeg generated a new video file, so update the video duration
- // See https://trac.ffmpeg.org/ticket/5456
- video.duration = await getVideoStreamDuration(videoTranscodedPath)
- await video.save()
+ try {
+ await transcodeVOD(transcodeOptions)
- return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
- })
+ await remove(audioInputPath)
+ await remove(tmpPreviewPath)
+ } catch (err) {
+ await remove(tmpPreviewPath)
+ throw err
+ }
+
+ // Important to do this before getVideoFilename() to take in account the new file extension
+ inputVideoFile.extname = newExtname
+ inputVideoFile.resolution = resolution
+ inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
+
+ // ffmpeg generated a new video file, so update the video duration
+ // See https://trac.ffmpeg.org/ticket/5456
+ video.duration = await getVideoStreamDuration(videoTranscodedPath)
+ await video.save()
+
+ return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
+ })
+
+ return result
+ } finally {
+ inputFileMutexReleaser()
+ }
}
// Concat TS segments from a live video to a fragmented mp4 HLS playlist
video: MVideo
concatenatedTsFilePath: string
resolution: VideoResolution
- isPortraitMode: boolean
isAAC: boolean
+ inputFileMutexReleaser: MutexInterface.Releaser
}) {
return generateHlsPlaylistCommon({
- video: options.video,
- resolution: options.resolution,
- isPortraitMode: options.isPortraitMode,
- inputPath: options.concatenatedTsFilePath,
type: 'hls-from-ts' as 'hls-from-ts',
- isAAC: options.isAAC
+ inputPath: options.concatenatedTsFilePath,
+
+ ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
})
}
videoInputPath: string
resolution: VideoResolution
copyCodecs: boolean
- isPortraitMode: boolean
+ inputFileMutexReleaser: MutexInterface.Releaser
job?: Job
}) {
return generateHlsPlaylistCommon({
- video: options.video,
- resolution: options.resolution,
- copyCodecs: options.copyCodecs,
- isPortraitMode: options.isPortraitMode,
- inputPath: options.videoInputPath,
type: 'hls' as 'hls',
- job: options.job
+ inputPath: options.videoInputPath,
+
+ ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
})
}
video: MVideoFullLight,
videoFile: MVideoFile,
transcodingPath: string,
- outputPath: string
+ newVideoFile: MVideoFile
) {
- const stats = await stat(transcodingPath)
- const fps = await getVideoStreamFPS(transcodingPath)
- const metadata = await buildFileMetadata(transcodingPath)
+ const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+
+ try {
+ await video.reload()
+
+ const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
+
+ const stats = await stat(transcodingPath)
+
+ const probe = await ffprobePromise(transcodingPath)
+ const fps = await getVideoStreamFPS(transcodingPath, probe)
+ const metadata = await buildFileMetadata(transcodingPath, probe)
- await move(transcodingPath, outputPath, { overwrite: true })
+ await move(transcodingPath, outputPath, { overwrite: true })
- videoFile.size = stats.size
- videoFile.fps = fps
- videoFile.metadata = metadata
+ videoFile.size = stats.size
+ videoFile.fps = fps
+ videoFile.metadata = metadata
- await createTorrentAndSetInfoHash(video, videoFile)
+ await createTorrentAndSetInfoHash(video, videoFile)
- await VideoFileModel.customUpsert(videoFile, 'video', undefined)
- video.VideoFiles = await video.$get('VideoFiles')
+ const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
+ if (oldFile) await video.removeWebTorrentFile(oldFile)
- return { video, videoFile }
+ await VideoFileModel.customUpsert(videoFile, 'video', undefined)
+ video.VideoFiles = await video.$get('VideoFiles')
+
+ return { video, videoFile }
+ } finally {
+ mutexReleaser()
+ }
}
async function generateHlsPlaylistCommon (options: {
video: MVideo
inputPath: string
resolution: VideoResolution
+
+ inputFileMutexReleaser: MutexInterface.Releaser
+
copyCodecs?: boolean
isAAC?: boolean
- isPortraitMode: boolean
job?: Job
}) {
- const { type, video, inputPath, resolution, copyCodecs, isPortraitMode, isAAC, job } = options
+ const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const videoTranscodedBasePath = join(transcodeDirectory, type)
resolution,
copyCodecs,
- isPortraitMode,
isAAC,
+ inputFileMutexReleaser,
+
hlsPlaylist: {
videoFilename
},
await transcodeVOD(transcodeOptions)
// Create or update the playlist
- const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
-
- if (!playlist.playlistFilename) {
- playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
- }
-
- if (!playlist.segmentsSha256Filename) {
- playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
- }
-
- playlist.p2pMediaLoaderInfohashes = []
- playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
-
- playlist.type = VideoStreamingPlaylistType.HLS
-
- await playlist.save()
+ const playlist = await retryTransactionWrapper(() => {
+ return sequelizeTypescript.transaction(async transaction => {
+ return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
+ })
+ })
- // Build the new playlist file
- const extname = extnameUtil(videoFilename)
const newVideoFile = new VideoFileModel({
resolution,
- extname,
+ extname: extnameUtil(videoFilename),
size: 0,
filename: videoFilename,
fps: -1,
videoStreamingPlaylistId: playlist.id
})
- const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
+ const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
- // Move files from tmp transcoded directory to the appropriate place
- await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
+ try {
+ // VOD transcoding is a long task, refresh video attributes
+ await video.reload()
- // Move playlist file
- const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
- await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
- // Move video file
- await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
+ const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
+ await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
- const stats = await stat(videoFilePath)
+ // Move playlist file
+ const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
+ await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
+ // Move video file
+ await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
- newVideoFile.size = stats.size
- newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
- newVideoFile.metadata = await buildFileMetadata(videoFilePath)
+ // Update video duration if it was not set (in case of a live for example)
+ if (!video.duration) {
+ video.duration = await getVideoStreamDuration(videoFilePath)
+ await video.save()
+ }
+
+ const stats = await stat(videoFilePath)
- await createTorrentAndSetInfoHash(playlist, newVideoFile)
+ newVideoFile.size = stats.size
+ newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
+ newVideoFile.metadata = await buildFileMetadata(videoFilePath)
- const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
+ await createTorrentAndSetInfoHash(playlist, newVideoFile)
- const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
- playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
- playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
+ const oldFile = await VideoFileModel.loadHLSFile({
+ playlistId: playlist.id,
+ fps: newVideoFile.fps,
+ resolution: newVideoFile.resolution
+ })
- await playlist.save()
+ if (oldFile) {
+ await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
+ await oldFile.destroy()
+ }
- video.setHLSPlaylist(playlist)
+ const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
- await updateMasterHLSPlaylist(video, playlistWithFiles)
- await updateSha256VODSegments(video, playlistWithFiles)
+ await updatePlaylistAfterFileChange(video, playlist)
+
+ return { resolutionPlaylistPath, videoFile: savedVideoFile }
+ } finally {
+ mutexReleaser()
+ }
+}
+
+function buildOriginalFileResolution (inputResolution: number) {
+ if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) {
+ return toEven(inputResolution)
+ }
+
+ const resolutions = computeResolutionsToTranscode({
+ input: inputResolution,
+ type: 'vod',
+ includeInput: false,
+ strictLower: false,
+ // We don't really care about the audio resolution in this context
+ hasAudio: true
+ })
+
+ if (resolutions.length === 0) {
+ return toEven(inputResolution)
+ }
- return { resolutionPlaylistPath, videoFile: savedVideoFile }
+ return Math.max(...resolutions)
}
projects / github / Chocobozzz / PeerTube.git / blobdiff