Add ability to run transcoding jobs

11 files changed, 56 insertions, 40 deletions

diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 8160e7949..d969549b8 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -37,7 +37,7 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl
   for (const file of playlist.VideoFiles) {
     const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
 
-    await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => {
+    await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
       const size = await getVideoStreamSize(videoFilePath)
 
       const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
@@ -69,10 +69,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP
   // For all the resolutions available for this video
   for (const file of playlist.VideoFiles) {
     const rangeHashes: { [range: string]: string } = {}
+    const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
 
-    await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => {
+    await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
 
-      return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => {
+      return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
         const playlistContent = await readFile(resolutionPlaylistPath)
         const ranges = getRangesFromPlaylist(playlistContent.toString())
 
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 4beca3d75..54a7c566b 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -56,16 +56,17 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
 
 async function moveHLSFiles (video: MVideoWithAllFiles) {
   for (const playlist of video.VideoStreamingPlaylists) {
+    const playlistWithVideo = playlist.withVideo(video)
 
     for (const file of playlist.VideoFiles) {
       if (file.storage !== VideoStorage.FILE_SYSTEM) continue
 
       // Resolution playlist
       const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
-      await storeHLSFile(playlist, video, playlistFilename)
+      await storeHLSFile(playlistWithVideo, playlistFilename)
 
       // Resolution fragmented file
-      const fileUrl = await storeHLSFile(playlist, video, file.filename)
+      const fileUrl = await storeHLSFile(playlistWithVideo, file.filename)
 
       const oldPath = join(getHLSDirectory(video), file.filename)
 
@@ -78,10 +79,12 @@ async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) {
   for (const playlist of video.VideoStreamingPlaylists) {
     if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
 
+    const playlistWithVideo = playlist.withVideo(video)
+
     // Master playlist
-    playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename)
+    playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename)
     // Sha256 segments file
-    playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename)
+    playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename)
 
     playlist.storage = VideoStorage.OBJECT_STORAGE
 
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 904ef2e3c..2d0798e12 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -14,7 +14,7 @@ import {
   VideoTranscodingPayload
 } from '../../../../shared'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
-import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
+import { computeLowerResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'
 import { CONFIG } from '../../../initializers/config'
 import { VideoModel } from '../../../models/video/video'
@@ -81,7 +81,7 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
 
   const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
 
-  await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => {
+  await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
     return generateHlsPlaylistResolution({
       video,
       videoInputPath,
@@ -135,7 +135,7 @@ async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodi
 // ---------------------------------------------------------------------------
 
 async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) {
-  if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
+  if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
     // Remove webtorrent files if not enabled
     for (const file of video.VideoFiles) {
       await video.removeWebTorrentFileAndTorrent(file)
@@ -232,6 +232,7 @@ async function createHlsJobIfEnabled (user: MUserId, payload: {
     isPortraitMode: payload.isPortraitMode,
     copyCodecs: payload.copyCodecs,
     isMaxQuality: payload.isMaxQuality,
+    autoDeleteWebTorrentIfNeeded: true,
     isNewVideo: payload.isNewVideo
   }
 
@@ -261,7 +262,7 @@ async function createLowerResolutionsJobs (options: {
   const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options
 
   // Create transcoding jobs if there are enabled resolutions
-  const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod')
+  const resolutionsEnabled = computeLowerResolutionsToTranscode(videoFileResolution, 'vod')
   const resolutionCreated: string[] = []
 
   for (const resolution of resolutionsEnabled) {
@@ -288,6 +289,7 @@ async function createLowerResolutionsJobs (options: {
         isPortraitMode,
         copyCodecs: false,
         isMaxQuality: false,
+        autoDeleteWebTorrentIfNeeded: true,
         isNewVideo
       }
 
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 2562edb75..b3bf5a999 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -3,7 +3,7 @@ import { readFile } from 'fs-extra'
 import { createServer, Server } from 'net'
 import { createServer as createServerTLS, Server as ServerTLS } from 'tls'
 import {
-  computeResolutionsToTranscode,
+  computeLowerResolutionsToTranscode,
   ffprobePromise,
   getVideoFileBitrate,
   getVideoFileFPS,
@@ -402,7 +402,7 @@ class LiveManager {
 
   private buildAllResolutionsToTranscode (originResolution: number) {
     const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
-      ? computeResolutionsToTranscode(originResolution, 'live')
+      ? computeLowerResolutionsToTranscode(originResolution, 'live')
       : []
 
     return resolutionsEnabled.concat([ originResolution ])
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts
index 12acb3aec..4f17073f4 100644
--- a/server/lib/object-storage/keys.ts
+++ b/server/lib/object-storage/keys.ts
@@ -1,12 +1,12 @@
 import { join } from 'path'
-import { MStreamingPlaylist, MVideoUUID } from '@server/types/models'
+import { MStreamingPlaylistVideo } from '@server/types/models'
 
-function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
-  return join(generateHLSObjectBaseStorageKey(playlist, video), filename)
+function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
+  return join(generateHLSObjectBaseStorageKey(playlist), filename)
 }
 
-function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) {
-  return join(playlist.getStringType(), video.uuid)
+function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
+  return join(playlist.getStringType(), playlist.Video.uuid)
 }
 
 function generateWebTorrentObjectStorageKey (filename: string) {
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 15b8f58d5..8988f3e2a 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -1,17 +1,17 @@
 import { join } from 'path'
 import { logger } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
-import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models'
+import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
 import { getHLSDirectory } from '../paths'
 import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
 import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
 
-function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
-  const baseHlsDirectory = getHLSDirectory(video)
+function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) {
+  const baseHlsDirectory = getHLSDirectory(playlist.Video)
 
   return storeObject({
     inputPath: join(baseHlsDirectory, filename),
-    objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename),
+    objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
     bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
   })
 }
@@ -24,16 +24,16 @@ function storeWebTorrentFile (filename: string) {
   })
 }
 
-function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) {
-  return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
+  return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
 }
 
 function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
   return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
 }
 
-async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) {
-  const key = generateHLSObjectStorageKey(playlist, video, filename)
+async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
+  const key = generateHLSObjectStorageKey(playlist, filename)
 
   logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
 
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index d2384f53c..36270e5c1 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -115,7 +115,7 @@ function generateVideoMiniature (options: {
 }) {
   const { video, videoFile, type } = options
 
-  return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => {
+  return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), input => {
     const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
 
     const thumbnailCreator = videoFile.isAudio()
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index 250a678eb..d0db05216 100644
--- a/server/lib/transcoding/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -35,7 +35,7 @@ function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVid
   const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
   const newExtname = '.mp4'
 
-  return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => {
+  return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => {
     const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
 
     const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath)
@@ -81,7 +81,7 @@ function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: V
   const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
   const extname = '.mp4'
 
-  return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => {
+  return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => {
     const newVideoFile = new VideoFileModel({
       resolution,
       extname,
@@ -134,7 +134,7 @@ function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolutio
 
   const inputVideoFile = video.getMinQualityFile()
 
-  return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => {
+  return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => {
     const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
 
     // If the user updates the video preview during transcoding
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
index 4c5d0c89d..27058005c 100644
--- a/server/lib/video-path-manager.ts
+++ b/server/lib/video-path-manager.ts
@@ -3,7 +3,14 @@ import { extname, join } from 'path'
 import { buildUUID } from '@server/helpers/uuid'
 import { extractVideo } from '@server/helpers/video'
 import { CONFIG } from '@server/initializers/config'
-import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
+import {
+  MStreamingPlaylistVideo,
+  MVideo,
+  MVideoFile,
+  MVideoFileStreamingPlaylistVideo,
+  MVideoFileVideo,
+  MVideoUUID
+} from '@server/types/models'
 import { VideoStorage } from '@shared/models'
 import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
 import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
@@ -43,10 +50,10 @@ class VideoPathManager {
     return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename)
   }
 
-  async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
+  async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
     if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
       return this.makeAvailableFactory(
-        () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile),
+        () => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile),
         false,
         cb
       )
@@ -55,10 +62,10 @@ class VideoPathManager {
     const destination = this.buildTMPDestination(videoFile.filename)
 
     if (videoFile.isHLS()) {
-      const video = extractVideo(videoOrPlaylist)
+      const playlist = (videoFile as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 
       return this.makeAvailableFactory(
-        () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination),
+        () => makeHLSFileAvailable(playlist, videoFile.filename, destination),
         true,
         cb
       )
@@ -71,19 +78,20 @@ class VideoPathManager {
     )
   }
 
-  async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
+  async makeAvailableResolutionPlaylistFile <T> (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
     const filename = getHlsResolutionPlaylistFilename(videoFile.filename)
 
     if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
       return this.makeAvailableFactory(
-        () => join(getHLSDirectory(playlist.Video), filename),
+        () => join(getHLSDirectory(videoFile.getVideo()), filename),
         false,
         cb
       )
     }
 
+    const playlist = videoFile.VideoStreamingPlaylist
     return this.makeAvailableFactory(
-      () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
+      () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)),
       true,
       cb
     )
@@ -99,7 +107,7 @@ class VideoPathManager {
     }
 
     return this.makeAvailableFactory(
-      () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
+      () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)),
       true,
       cb
     )
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index 0b51f5c6b..bf6dd4bc8 100644
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -80,6 +80,8 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
 }
 
 function moveToFailedTranscodingState (video: MVideoFullLight) {
+  if (video.state === VideoState.TRANSCODING_FAILED) return
+
   return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
 }
 
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 0a2b93cc0..1cfe4f27c 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -105,7 +105,7 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF
   return addTranscodingJob(dataInput, jobOptions)
 }
 
-async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) {
+async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions = {}) {
   await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
 
   return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options)