From d7a25329f9e607894d29ab342b9cb66638b56dc0 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 15 Nov 2019 15:06:03 +0100 Subject: Add ability to disable webtorrent In favour of HLS --- server/lib/job-queue/handlers/video-file-import.ts | 6 +- server/lib/job-queue/handlers/video-import.ts | 12 +-- server/lib/job-queue/handlers/video-transcoding.ts | 119 +++++++++++---------- 3 files changed, 74 insertions(+), 63 deletions(-) (limited to 'server/lib/job-queue') diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 5c5b7dccb..99c991e72 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -7,6 +7,8 @@ import { copy, stat } from 'fs-extra' import { VideoFileModel } from '../../../models/video/video-file' import { extname } from 'path' import { MVideoFile, MVideoWithFile } from '@server/typings/models' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { getVideoFilePath } from '@server/lib/video-paths' export type VideoFileImportPayload = { videoUUID: string, @@ -68,10 +70,10 @@ async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) { updatedVideoFile = currentVideoFile } - const outputPath = video.getVideoFilePath(updatedVideoFile) + const outputPath = getVideoFilePath(video, updatedVideoFile) await copy(inputFilePath, outputPath) - await video.createTorrentAndSetInfoHash(updatedVideoFile) + await createTorrentAndSetInfoHash(video, updatedVideoFile) await updatedVideoFile.save() diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 93a3e9d90..1fca17584 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -4,14 +4,14 @@ import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' import { VideoImportModel } from '../../../models/video/video-import' import { VideoImportState } from '../../../../shared/models/videos' import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' -import { extname, join } from 'path' +import { extname } from 'path' import { VideoFileModel } from '../../../models/video/video-file' import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' import { VideoState } from '../../../../shared' import { JobQueue } from '../index' import { federateVideoIfNeeded } from '../../activitypub' import { VideoModel } from '../../../models/video/video' -import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' +import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' import { getSecureTorrentName } from '../../../helpers/utils' import { move, remove, stat } from 'fs-extra' import { Notifier } from '../../notifier' @@ -21,7 +21,7 @@ import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumb import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' import { MThumbnail } from '../../../typings/models/video/thumbnail' import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import' -import { MVideoBlacklistVideo, MVideoBlacklist } from '@server/typings/models' +import { getVideoFilePath } from '@server/lib/video-paths' type VideoImportYoutubeDLPayload = { type: 'youtube-dl' @@ -142,12 +142,12 @@ async function processFile (downloader: () => Promise, videoImport: MVid } videoFile = new VideoFileModel(videoFileData) - const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ] }) + const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) // To clean files if the import fails const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) // Move file - videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImportWithFiles.Video.getVideoFilename(videoFile)) + videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile) await move(tempVideoPath, videoDestFile) tempVideoPath = null // This path is not used anymore @@ -168,7 +168,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid } // Create torrent - await videoImportWithFiles.Video.createTorrentAndSetInfoHash(videoFile) + await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) const { videoImportUpdated, video } = await sequelizeTypescript.transaction(async t => { const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 2ebe15bcb..39b9fac98 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -1,5 +1,5 @@ import * as Bull from 'bull' -import { VideoResolution, VideoState } from '../../../../shared' +import { VideoResolution } from '../../../../shared' import { logger } from '../../../helpers/logger' import { VideoModel } from '../../../models/video/video' import { JobQueue } from '../job-queue' @@ -8,10 +8,10 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import * as Bluebird from 'bluebird' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' -import { generateHlsPlaylist, optimizeVideofile, transcodeOriginalVideofile, mergeAudioVideofile } from '../../video-transcoding' +import { generateHlsPlaylist, mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewResolution } from '../../video-transcoding' import { Notifier } from '../../notifier' import { CONFIG } from '../../../initializers/config' -import { MVideoUUID, MVideoWithFile } from '@server/typings/models' +import { MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/typings/models' interface BaseTranscodingPayload { videoUUID: string @@ -22,6 +22,7 @@ interface HLSTranscodingPayload extends BaseTranscodingPayload { type: 'hls' isPortraitMode?: boolean resolution: VideoResolution + copyCodecs: boolean } interface NewResolutionTranscodingPayload extends BaseTranscodingPayload { @@ -54,11 +55,11 @@ async function processVideoTranscoding (job: Bull.Job) { } if (payload.type === 'hls') { - await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) + await generateHlsPlaylist(video, payload.resolution, payload.copyCodecs, payload.isPortraitMode || false) await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) } else if (payload.type === 'new-resolution') { - await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) + await transcodeNewResolution(video, payload.resolution, payload.isPortraitMode || false) await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload) } else if (payload.type === 'merge-audio') { @@ -66,7 +67,7 @@ async function processVideoTranscoding (job: Bull.Job) { await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload) } else { - await optimizeVideofile(video) + await optimizeOriginalVideofile(video) await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) } @@ -74,48 +75,24 @@ async function processVideoTranscoding (job: Bull.Job) { return video } -async function onHlsPlaylistGenerationSuccess (video: MVideoUUID) { +async function onHlsPlaylistGenerationSuccess (video: MVideoFullLight) { if (video === undefined) return undefined - await sequelizeTypescript.transaction(async t => { - // Maybe the video changed in database, refresh it - let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) - // Video does not exist anymore - if (!videoDatabase) return undefined - - // If the video was not published, we consider it is a new one for other instances - await federateVideoIfNeeded(videoDatabase, false, t) - }) -} - -async function publishNewResolutionIfNeeded (video: MVideoUUID, payload?: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload) { - const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { - // Maybe the video changed in database, refresh it - let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) - // Video does not exist anymore - if (!videoDatabase) return undefined - - let videoPublished = false - - // We transcoded the video file in another format, now we can publish it - if (videoDatabase.state !== VideoState.PUBLISHED) { - videoPublished = true - - videoDatabase.state = VideoState.PUBLISHED - videoDatabase.publishedAt = new Date() - videoDatabase = await videoDatabase.save({ transaction: t }) + // We generated the HLS playlist, we don't need the webtorrent files anymore if the admin disabled it + if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { + for (const file of video.VideoFiles) { + await video.removeFile(file) + await file.destroy() } - // If the video was not published, we consider it is a new one for other instances - await federateVideoIfNeeded(videoDatabase, videoPublished, t) + video.VideoFiles = [] + } - return { videoDatabase, videoPublished } - }) + return publishAndFederateIfNeeded(video) +} - if (videoPublished) { - Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) - Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) - } +async function publishNewResolutionIfNeeded (video: MVideoUUID, payload?: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload) { + await publishAndFederateIfNeeded(video) await createHlsJobIfEnabled(payload) } @@ -124,7 +101,7 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O if (videoArg === undefined) return undefined // Outside the transaction (IO on disk) - const { videoFileResolution } = await videoArg.getOriginalFileResolution() + const { videoFileResolution } = await videoArg.getMaxQualityResolution() const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it @@ -141,14 +118,29 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O let videoPublished = false + const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getMaxQualityFile().resolution }) + await createHlsJobIfEnabled(hlsPayload) + if (resolutionsEnabled.length !== 0) { const tasks: (Bluebird> | Promise>)[] = [] for (const resolution of resolutionsEnabled) { - const dataInput = { - type: 'new-resolution' as 'new-resolution', - videoUUID: videoDatabase.uuid, - resolution + let dataInput: VideoTranscodingPayload + + if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) { + dataInput = { + type: 'new-resolution' as 'new-resolution', + videoUUID: videoDatabase.uuid, + resolution + } + } else if (CONFIG.TRANSCODING.HLS.ENABLED) { + dataInput = { + type: 'hls', + videoUUID: videoDatabase.uuid, + resolution, + isPortraitMode: false, + copyCodecs: false + } } const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }) @@ -159,11 +151,8 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) } else { - videoPublished = true - // No transcoding to do, it's now published - videoDatabase.state = VideoState.PUBLISHED - videoDatabase = await videoDatabase.save({ transaction: t }) + videoPublished = await videoDatabase.publishIfNeededAndSave(t) logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) } @@ -175,9 +164,6 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) - - const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }) - await createHlsJobIfEnabled(hlsPayload) } // --------------------------------------------------------------------------- @@ -196,9 +182,32 @@ function createHlsJobIfEnabled (payload?: { videoUUID: string, resolution: numbe type: 'hls' as 'hls', videoUUID: payload.videoUUID, resolution: payload.resolution, - isPortraitMode: payload.isPortraitMode + isPortraitMode: payload.isPortraitMode, + copyCodecs: true } return JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }) } } + +async function publishAndFederateIfNeeded (video: MVideoUUID) { + const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { + // Maybe the video changed in database, refresh it + const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) + // Video does not exist anymore + if (!videoDatabase) return undefined + + // We transcoded the video file in another format, now we can publish it + const videoPublished = await videoDatabase.publishIfNeededAndSave(t) + + // If the video was not published, we consider it is a new one for other instances + await federateVideoIfNeeded(videoDatabase, videoPublished, t) + + return { videoDatabase, videoPublished } + }) + + if (videoPublished) { + Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) + Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) + } +} -- cgit v1.2.3