From 3545e72c686ff1725bbdfd8d16d693e2f4aa75a3 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 12 Oct 2022 16:09:02 +0200 Subject: Put private videos under a specific subdirectory --- .../lib/job-queue/handlers/manage-video-torrent.ts | 2 +- .../job-queue/handlers/move-to-object-storage.ts | 6 +- server/lib/job-queue/handlers/video-live-ending.ts | 22 +++-- server/lib/job-queue/handlers/video-transcoding.ts | 95 +++++++++++++--------- 4 files changed, 75 insertions(+), 50 deletions(-) (limited to 'server/lib/job-queue') diff --git a/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/lib/job-queue/handlers/manage-video-torrent.ts index 03aa414c9..425915c96 100644 --- a/server/lib/job-queue/handlers/manage-video-torrent.ts +++ b/server/lib/job-queue/handlers/manage-video-torrent.ts @@ -82,7 +82,7 @@ async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) { async function loadFileOrLog (videoFileId: number) { if (!videoFileId) return undefined - const file = await VideoFileModel.loadWithVideo(videoFileId) + const file = await VideoFileModel.load(videoFileId) if (!file) { logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index 28c3d325d..0b68555d1 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts @@ -3,10 +3,10 @@ import { remove } from 'fs-extra' import { join } from 'path' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { updateTorrentMetadata } from '@server/helpers/webtorrent' -import { CONFIG } from '@server/initializers/config' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' +import { VideoPathManager } from '@server/lib/video-path-manager' import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' import { VideoModel } from '@server/models/video/video' import { VideoJobInfoModel } from '@server/models/video/video-job-info' @@ -72,9 +72,9 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) { for (const file of video.VideoFiles) { if (file.storage !== VideoStorage.FILE_SYSTEM) continue - const fileUrl = await storeWebTorrentFile(file.filename) + const fileUrl = await storeWebTorrentFile(video, file) - const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) + const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) } } diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 7dbffc955..c6263f55a 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -18,6 +18,7 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' import { logger, loggerTagsFactory } from '../../../helpers/logger' +import { VideoPathManager } from '@server/lib/video-path-manager' const lTags = loggerTagsFactory('live', 'job') @@ -205,18 +206,27 @@ async function assignReplayFilesToVideo (options: { const concatenatedTsFiles = await readdir(replayDirectory) for (const concatenatedTsFile of concatenatedTsFiles) { + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) const probe = await ffprobePromise(concatenatedTsFilePath) const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) - await generateHlsPlaylistResolutionFromTS({ - video, - concatenatedTsFilePath, - resolution, - isAAC: audioStream?.codec_name === 'aac' - }) + try { + await generateHlsPlaylistResolutionFromTS({ + video, + inputFileMutexReleaser, + concatenatedTsFilePath, + resolution, + isAAC: audioStream?.codec_name === 'aac' + }) + } catch (err) { + logger.error('Cannot generate HLS playlist resolution from TS files.', { err }) + } + + inputFileMutexReleaser() } return video diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index b0e92acf7..48c675678 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -94,15 +94,24 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() - await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { - return generateHlsPlaylistResolution({ - video, - videoInputPath, - resolution: payload.resolution, - copyCodecs: payload.copyCodecs, - job + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await videoFileInput.getVideo().reload() + + await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { + return generateHlsPlaylistResolution({ + video, + videoInputPath, + inputFileMutexReleaser, + resolution: payload.resolution, + copyCodecs: payload.copyCodecs, + job + }) }) - }) + } finally { + inputFileMutexReleaser() + } logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) @@ -177,38 +186,44 @@ async function onVideoFirstWebTorrentTranscoding ( transcodeType: TranscodeVODOptionsType, user: MUserId ) { - const { resolution, audioStream } = await videoArg.probeMaxQualityFile() - - // Maybe the video changed in database, refresh it - const videoDatabase = await VideoModel.loadFull(videoArg.uuid) - // Video does not exist anymore - if (!videoDatabase) return undefined - - // Generate HLS version of the original file - const originalFileHLSPayload = { - ...payload, - - hasAudio: !!audioStream, - resolution: videoDatabase.getMaxQualityFile().resolution, - // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues - copyCodecs: transcodeType !== 'quick-transcode', - isMaxQuality: true - } - const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) - const hasNewResolutions = await createLowerResolutionsJobs({ - video: videoDatabase, - user, - videoFileResolution: resolution, - hasAudio: !!audioStream, - type: 'webtorrent', - isNewVideo: payload.isNewVideo ?? true - }) - - await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') - - // Move to next state if there are no other resolutions to generate - if (!hasHls && !hasNewResolutions) { - await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) + const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) + + try { + // Maybe the video changed in database, refresh it + const videoDatabase = await VideoModel.loadFull(videoArg.uuid) + // Video does not exist anymore + if (!videoDatabase) return undefined + + const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile() + + // Generate HLS version of the original file + const originalFileHLSPayload = { + ...payload, + + hasAudio: !!audioStream, + resolution: videoDatabase.getMaxQualityFile().resolution, + // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues + copyCodecs: transcodeType !== 'quick-transcode', + isMaxQuality: true + } + const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) + const hasNewResolutions = await createLowerResolutionsJobs({ + video: videoDatabase, + user, + videoFileResolution: resolution, + hasAudio: !!audioStream, + type: 'webtorrent', + isNewVideo: payload.isNewVideo ?? true + }) + + await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') + + // Move to next state if there are no other resolutions to generate + if (!hasHls && !hasNewResolutions) { + await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) + } + } finally { + mutexReleaser() } } -- cgit v1.2.3