Fix HLS re transcoding with object storage enabled

5 files changed, 118 insertions, 18 deletions

diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 1574ff27b..985f50587 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -3,6 +3,7 @@ import { flatten, uniq } from 'lodash'
 import { basename, dirname, join } from 'path'
 import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
 import { sha256 } from '@shared/extra-utils'
+import { VideoStorage } from '@shared/models'
 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
 import { logger } from '../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
@@ -12,6 +13,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
 import { sequelizeTypescript } from '../initializers/database'
 import { VideoFileModel } from '../models/video/video-file'
 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
+import { storeHLSFile } from './object-storage'
 import { getHlsResolutionPlaylistFilename } from './paths'
 import { VideoPathManager } from './video-path-manager'
 
@@ -58,8 +60,12 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl
     })
   }
 
-  await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
-    return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+  await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => {
+    await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+
+    if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
+      await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath)
+    }
   })
 }
 
@@ -94,6 +100,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP
 
   const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
   await outputJSON(outputPath, json)
+
+  if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
+    await storeHLSFile(playlist, playlist.segmentsSha256Filename)
+    await remove(outputPath)
+  }
 }
 
 async function buildSha256Segment (segmentPath: string) {
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 9e39322a8..69b441176 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -1,7 +1,7 @@
 import { Job } from 'bull'
 import { remove } from 'fs-extra'
 import { join } from 'path'
-import { logger } from '@server/helpers/logger'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { updateTorrentMetadata } from '@server/helpers/webtorrent'
 import { CONFIG } from '@server/initializers/config'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
@@ -13,6 +13,8 @@ import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
 import { MoveObjectStoragePayload, VideoStorage } from '@shared/models'
 
+const lTagsBase = loggerTagsFactory('move-object-storage')
+
 export async function processMoveToObjectStorage (job: Job) {
   const payload = job.data as MoveObjectStoragePayload
   logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
@@ -20,26 +22,33 @@ export async function processMoveToObjectStorage (job: Job) {
   const video = await VideoModel.loadWithFiles(payload.videoUUID)
   // No video, maybe deleted?
   if (!video) {
-    logger.info('Can\'t process job %d, video does not exist.', job.id)
+    logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
     return undefined
   }
 
+  const lTags = lTagsBase(video.uuid, video.url)
+
   try {
     if (video.VideoFiles) {
+      logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags)
+
       await moveWebTorrentFiles(video)
     }
 
     if (video.VideoStreamingPlaylists) {
+      logger.debug('Moving HLS playlist of %s.', video.uuid)
+
       await moveHLSFiles(video)
     }
 
     const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
     if (pendingMove === 0) {
-      logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
+      logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id, lTags)
+
       await doAfterLastJob(video, payload.isNewVideo)
     }
   } catch (err) {
-    logger.error('Cannot move video %s to object storage.', video.url, { err })
+    logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTags })
 
     await moveToFailedMoveToObjectStorageState(video)
     await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 8988f3e2a..066b48ab0 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -6,11 +6,9 @@ import { getHLSDirectory } from '../paths'
 import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
 import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
 
-function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) {
-  const baseHlsDirectory = getHLSDirectory(playlist.Video)
-
+function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) {
   return storeObject({
-    inputPath: join(baseHlsDirectory, filename),
+    inputPath: path ?? join(getHLSDirectory(playlist.Video), filename),
     objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
     bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
   })
diff --git a/server/tests/api/videos/video-create-transcoding.ts b/server/tests/api/videos/video-create-transcoding.ts
index dcdbd9c6e..445866a16 100644
--- a/server/tests/api/videos/video-create-transcoding.ts
+++ b/server/tests/api/videos/video-create-transcoding.ts
@@ -2,11 +2,12 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { expectStartWith } from '@server/tests/shared'
+import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared'
 import { areObjectStorageTestsDisabled } from '@shared/core-utils'
 import { HttpStatusCode, VideoDetails } from '@shared/models'
 import {
   cleanupTests,
+  ConfigCommand,
   createMultipleServers,
   doubleFollow,
   expectNoFailedTranscodingJob,
@@ -25,14 +26,19 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
     await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
   }
 
-  const streamingPlaylistFiles = video.streamingPlaylists.length === 0
-    ? []
-    : video.streamingPlaylists[0].files
+  if (video.streamingPlaylists.length === 0) return
 
-  for (const file of streamingPlaylistFiles) {
+  const hlsPlaylist = video.streamingPlaylists[0]
+  for (const file of hlsPlaylist.files) {
     expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
     await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
   }
+
+  expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl())
+  await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
+
+  expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl())
+  await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
 }
 
 function runTests (objectStorage: boolean) {
@@ -150,6 +156,75 @@ function runTests (objectStorage: boolean) {
     }
   })
 
+  it('Should correctly update HLS playlist on resolution change', async function () {
+    await servers[0].config.updateExistingSubConfig({
+      newConfig: {
+        transcoding: {
+          enabled: true,
+          resolutions: ConfigCommand.getCustomConfigResolutions(false),
+
+          webtorrent: {
+            enabled: true
+          },
+          hls: {
+            enabled: true
+          }
+        }
+      }
+    })
+
+    const { uuid } = await servers[0].videos.quickUpload({ name: 'quick' })
+
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      const videoDetails = await server.videos.get({ id: uuid })
+
+      expect(videoDetails.files).to.have.lengthOf(1)
+      expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
+      expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1)
+
+      if (objectStorage) await checkFilesInObjectStorage(videoDetails)
+    }
+
+    await servers[0].config.updateExistingSubConfig({
+      newConfig: {
+        transcoding: {
+          enabled: true,
+          resolutions: ConfigCommand.getCustomConfigResolutions(true),
+
+          webtorrent: {
+            enabled: true
+          },
+          hls: {
+            enabled: true
+          }
+        }
+      }
+    })
+
+    await servers[0].videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      const videoDetails = await server.videos.get({ id: uuid })
+
+      expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
+      expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
+
+      if (objectStorage) {
+        await checkFilesInObjectStorage(videoDetails)
+
+        const hlsPlaylist = videoDetails.streamingPlaylists[0]
+        const resolutions = hlsPlaylist.files.map(f => f.resolution.id)
+        await checkResolutionsInMasterPlaylist({ server: servers[0], playlistUrl: hlsPlaylist.playlistUrl, resolutions })
+
+        const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url })
+        expect(Object.keys(shaBody)).to.have.lengthOf(5)
+      }
+    }
+  })
+
   it('Should not have updated published at attributes', async function () {
     const video = await servers[0].videos.get({ id: videoUUID })
 
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts
index c85130fef..b90e9bde9 100644
--- a/server/tests/cli/create-transcoding-job.ts
+++ b/server/tests/cli/create-transcoding-job.ts
@@ -14,7 +14,7 @@ import {
   setAccessTokensToServers,
   waitJobs
 } from '@shared/server-commands'
-import { expectStartWith } from '../shared'
+import { checkResolutionsInMasterPlaylist, expectStartWith } from '../shared'
 
 const expect = chai.expect
 
@@ -163,11 +163,18 @@ function runTests (objectStorage: boolean) {
 
       expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
 
-      const files = videoDetails.streamingPlaylists[0].files
+      const hlsPlaylist = videoDetails.streamingPlaylists[0]
+
+      const files = hlsPlaylist.files
       expect(files).to.have.lengthOf(1)
       expect(files[0].resolution.id).to.equal(480)
 
-      if (objectStorage) await checkFilesInObjectStorage(files, 'playlist')
+      if (objectStorage) {
+        await checkFilesInObjectStorage(files, 'playlist')
+
+        const resolutions = files.map(f => f.resolution.id)
+        await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
+      }
     }
   })