Fix HLS re transcoding with object storage enabled

3 files changed, 28 insertions, 10 deletions

diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 1574ff27b..985f50587 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -3,6 +3,7 @@ import { flatten, uniq } from 'lodash'
 import { basename, dirname, join } from 'path'
 import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
 import { sha256 } from '@shared/extra-utils'
+import { VideoStorage } from '@shared/models'
 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
 import { logger } from '../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
@@ -12,6 +13,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
 import { sequelizeTypescript } from '../initializers/database'
 import { VideoFileModel } from '../models/video/video-file'
 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
+import { storeHLSFile } from './object-storage'
 import { getHlsResolutionPlaylistFilename } from './paths'
 import { VideoPathManager } from './video-path-manager'
 
@@ -58,8 +60,12 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl
     })
   }
 
-  await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
-    return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+  await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => {
+    await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+
+    if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
+      await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath)
+    }
   })
 }
 
@@ -94,6 +100,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP
 
   const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
   await outputJSON(outputPath, json)
+
+  if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
+    await storeHLSFile(playlist, playlist.segmentsSha256Filename)
+    await remove(outputPath)
+  }
 }
 
 async function buildSha256Segment (segmentPath: string) {
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 9e39322a8..69b441176 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -1,7 +1,7 @@
 import { Job } from 'bull'
 import { remove } from 'fs-extra'
 import { join } from 'path'
-import { logger } from '@server/helpers/logger'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { updateTorrentMetadata } from '@server/helpers/webtorrent'
 import { CONFIG } from '@server/initializers/config'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
@@ -13,6 +13,8 @@ import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
 import { MoveObjectStoragePayload, VideoStorage } from '@shared/models'
 
+const lTagsBase = loggerTagsFactory('move-object-storage')
+
 export async function processMoveToObjectStorage (job: Job) {
   const payload = job.data as MoveObjectStoragePayload
   logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
@@ -20,26 +22,33 @@ export async function processMoveToObjectStorage (job: Job) {
   const video = await VideoModel.loadWithFiles(payload.videoUUID)
   // No video, maybe deleted?
   if (!video) {
-    logger.info('Can\'t process job %d, video does not exist.', job.id)
+    logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
     return undefined
   }
 
+  const lTags = lTagsBase(video.uuid, video.url)
+
   try {
     if (video.VideoFiles) {
+      logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags)
+
       await moveWebTorrentFiles(video)
     }
 
     if (video.VideoStreamingPlaylists) {
+      logger.debug('Moving HLS playlist of %s.', video.uuid)
+
       await moveHLSFiles(video)
     }
 
     const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
     if (pendingMove === 0) {
-      logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
+      logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id, lTags)
+
       await doAfterLastJob(video, payload.isNewVideo)
     }
   } catch (err) {
-    logger.error('Cannot move video %s to object storage.', video.url, { err })
+    logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTags })
 
     await moveToFailedMoveToObjectStorageState(video)
     await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 8988f3e2a..066b48ab0 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -6,11 +6,9 @@ import { getHLSDirectory } from '../paths'
 import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
 import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
 
-function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) {
-  const baseHlsDirectory = getHLSDirectory(playlist.Video)
-
+function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) {
   return storeObject({
-    inputPath: join(baseHlsDirectory, filename),
+    inputPath: path ?? join(getHLSDirectory(playlist.Video), filename),
     objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
     bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
   })