Add ability to disable webtorrent

In favour of HLS

3 files changed, 74 insertions, 63 deletions

diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 5c5b7dccb..99c991e72 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -7,6 +7,8 @@ import { copy, stat } from 'fs-extra'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { extname } from 'path'
 import { MVideoFile, MVideoWithFile } from '@server/typings/models'
+import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
+import { getVideoFilePath } from '@server/lib/video-paths'
 
 export type VideoFileImportPayload = {
   videoUUID: string,
@@ -68,10 +70,10 @@ async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) {
     updatedVideoFile = currentVideoFile
   }
 
-  const outputPath = video.getVideoFilePath(updatedVideoFile)
+  const outputPath = getVideoFilePath(video, updatedVideoFile)
   await copy(inputFilePath, outputPath)
 
-  await video.createTorrentAndSetInfoHash(updatedVideoFile)
+  await createTorrentAndSetInfoHash(video, updatedVideoFile)
 
   await updatedVideoFile.save()
 
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 93a3e9d90..1fca17584 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -4,14 +4,14 @@ import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
 import { VideoImportModel } from '../../../models/video/video-import'
 import { VideoImportState } from '../../../../shared/models/videos'
 import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
-import { extname, join } from 'path'
+import { extname } from 'path'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
 import { VideoState } from '../../../../shared'
 import { JobQueue } from '../index'
 import { federateVideoIfNeeded } from '../../activitypub'
 import { VideoModel } from '../../../models/video/video'
-import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
+import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
 import { getSecureTorrentName } from '../../../helpers/utils'
 import { move, remove, stat } from 'fs-extra'
 import { Notifier } from '../../notifier'
@@ -21,7 +21,7 @@ import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumb
 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
 import { MThumbnail } from '../../../typings/models/video/thumbnail'
 import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
-import { MVideoBlacklistVideo, MVideoBlacklist } from '@server/typings/models'
+import { getVideoFilePath } from '@server/lib/video-paths'
 
 type VideoImportYoutubeDLPayload = {
   type: 'youtube-dl'
@@ -142,12 +142,12 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     }
     videoFile = new VideoFileModel(videoFileData)
 
-    const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ] })
+    const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
     // To clean files if the import fails
     const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
 
     // Move file
-    videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImportWithFiles.Video.getVideoFilename(videoFile))
+    videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile)
     await move(tempVideoPath, videoDestFile)
     tempVideoPath = null // This path is not used anymore
 
@@ -168,7 +168,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     }
 
     // Create torrent
-    await videoImportWithFiles.Video.createTorrentAndSetInfoHash(videoFile)
+    await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
 
     const { videoImportUpdated, video } = await sequelizeTypescript.transaction(async t => {
       const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 2ebe15bcb..39b9fac98 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -1,5 +1,5 @@
 import * as Bull from 'bull'
-import { VideoResolution, VideoState } from '../../../../shared'
+import { VideoResolution } from '../../../../shared'
 import { logger } from '../../../helpers/logger'
 import { VideoModel } from '../../../models/video/video'
 import { JobQueue } from '../job-queue'
@@ -8,10 +8,10 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { sequelizeTypescript } from '../../../initializers'
 import * as Bluebird from 'bluebird'
 import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
-import { generateHlsPlaylist, optimizeVideofile, transcodeOriginalVideofile, mergeAudioVideofile } from '../../video-transcoding'
+import { generateHlsPlaylist, mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewResolution } from '../../video-transcoding'
 import { Notifier } from '../../notifier'
 import { CONFIG } from '../../../initializers/config'
-import { MVideoUUID, MVideoWithFile } from '@server/typings/models'
+import { MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/typings/models'
 
 interface BaseTranscodingPayload {
   videoUUID: string
@@ -22,6 +22,7 @@ interface HLSTranscodingPayload extends BaseTranscodingPayload {
   type: 'hls'
   isPortraitMode?: boolean
   resolution: VideoResolution
+  copyCodecs: boolean
 }
 
 interface NewResolutionTranscodingPayload extends BaseTranscodingPayload {
@@ -54,11 +55,11 @@ async function processVideoTranscoding (job: Bull.Job) {
   }
 
   if (payload.type === 'hls') {
-    await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false)
+    await generateHlsPlaylist(video, payload.resolution, payload.copyCodecs, payload.isPortraitMode || false)
 
     await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
   } else if (payload.type === 'new-resolution') {
-    await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
+    await transcodeNewResolution(video, payload.resolution, payload.isPortraitMode || false)
 
     await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload)
   } else if (payload.type === 'merge-audio') {
@@ -66,7 +67,7 @@ async function processVideoTranscoding (job: Bull.Job) {
 
     await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload)
   } else {
-    await optimizeVideofile(video)
+    await optimizeOriginalVideofile(video)
 
     await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
   }
@@ -74,48 +75,24 @@ async function processVideoTranscoding (job: Bull.Job) {
   return video
 }
 
-async function onHlsPlaylistGenerationSuccess (video: MVideoUUID) {
+async function onHlsPlaylistGenerationSuccess (video: MVideoFullLight) {
   if (video === undefined) return undefined
 
-  await sequelizeTypescript.transaction(async t => {
-    // Maybe the video changed in database, refresh it
-    let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
-    // Video does not exist anymore
-    if (!videoDatabase) return undefined
-
-    // If the video was not published, we consider it is a new one for other instances
-    await federateVideoIfNeeded(videoDatabase, false, t)
-  })
-}
-
-async function publishNewResolutionIfNeeded (video: MVideoUUID, payload?: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload) {
-  const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
-    // Maybe the video changed in database, refresh it
-    let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
-    // Video does not exist anymore
-    if (!videoDatabase) return undefined
-
-    let videoPublished = false
-
-    // We transcoded the video file in another format, now we can publish it
-    if (videoDatabase.state !== VideoState.PUBLISHED) {
-      videoPublished = true
-
-      videoDatabase.state = VideoState.PUBLISHED
-      videoDatabase.publishedAt = new Date()
-      videoDatabase = await videoDatabase.save({ transaction: t })
+  // We generated the HLS playlist, we don't need the webtorrent files anymore if the admin disabled it
+  if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
+    for (const file of video.VideoFiles) {
+      await video.removeFile(file)
+      await file.destroy()
     }
 
-    // If the video was not published, we consider it is a new one for other instances
-    await federateVideoIfNeeded(videoDatabase, videoPublished, t)
+    video.VideoFiles = []
+  }
 
-    return { videoDatabase, videoPublished }
-  })
+  return publishAndFederateIfNeeded(video)
+}
 
-  if (videoPublished) {
-    Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
-    Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
-  }
+async function publishNewResolutionIfNeeded (video: MVideoUUID, payload?: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload) {
+  await publishAndFederateIfNeeded(video)
 
   await createHlsJobIfEnabled(payload)
 }
@@ -124,7 +101,7 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
   if (videoArg === undefined) return undefined
 
   // Outside the transaction (IO on disk)
-  const { videoFileResolution } = await videoArg.getOriginalFileResolution()
+  const { videoFileResolution } = await videoArg.getMaxQualityResolution()
 
   const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
     // Maybe the video changed in database, refresh it
@@ -141,14 +118,29 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
 
     let videoPublished = false
 
+    const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getMaxQualityFile().resolution })
+    await createHlsJobIfEnabled(hlsPayload)
+
     if (resolutionsEnabled.length !== 0) {
       const tasks: (Bluebird<Bull.Job<any>> | Promise<Bull.Job<any>>)[] = []
 
       for (const resolution of resolutionsEnabled) {
-        const dataInput = {
-          type: 'new-resolution' as 'new-resolution',
-          videoUUID: videoDatabase.uuid,
-          resolution
+        let dataInput: VideoTranscodingPayload
+
+        if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) {
+          dataInput = {
+            type: 'new-resolution' as 'new-resolution',
+            videoUUID: videoDatabase.uuid,
+            resolution
+          }
+        } else if (CONFIG.TRANSCODING.HLS.ENABLED) {
+          dataInput = {
+            type: 'hls',
+            videoUUID: videoDatabase.uuid,
+            resolution,
+            isPortraitMode: false,
+            copyCodecs: false
+          }
         }
 
         const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
@@ -159,11 +151,8 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
 
       logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled })
     } else {
-      videoPublished = true
-
       // No transcoding to do, it's now published
-      videoDatabase.state = VideoState.PUBLISHED
-      videoDatabase = await videoDatabase.save({ transaction: t })
+      videoPublished = await videoDatabase.publishIfNeededAndSave(t)
 
       logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
     }
@@ -175,9 +164,6 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
 
   if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
   if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
-
-  const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })
-  await createHlsJobIfEnabled(hlsPayload)
 }
 
 // ---------------------------------------------------------------------------
@@ -196,9 +182,32 @@ function createHlsJobIfEnabled (payload?: { videoUUID: string, resolution: numbe
       type: 'hls' as 'hls',
       videoUUID: payload.videoUUID,
       resolution: payload.resolution,
-      isPortraitMode: payload.isPortraitMode
+      isPortraitMode: payload.isPortraitMode,
+      copyCodecs: true
     }
 
     return JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload })
   }
 }
+
+async function publishAndFederateIfNeeded (video: MVideoUUID) {
+  const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
+    // Maybe the video changed in database, refresh it
+    const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
+    // Video does not exist anymore
+    if (!videoDatabase) return undefined
+
+    // We transcoded the video file in another format, now we can publish it
+    const videoPublished = await videoDatabase.publishIfNeededAndSave(t)
+
+    // If the video was not published, we consider it is a new one for other instances
+    await federateVideoIfNeeded(videoDatabase, videoPublished, t)
+
+    return { videoDatabase, videoPublished }
+  })
+
+  if (videoPublished) {
+    Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
+    Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
+  }
+}