Don't stuck state when move transcoding job failed

2 files changed, 27 insertions, 13 deletions

diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index b5eea0184..d9c415b2d 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
 import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
-import { moveToNextState } from '@server/lib/video-state'
+import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
@@ -24,18 +24,25 @@ export async function processMoveToObjectStorage (job: Job) {
     return undefined
   }
 
-  if (video.VideoFiles) {
-    await moveWebTorrentFiles(video)
-  }
+  try {
+    if (video.VideoFiles) {
+      await moveWebTorrentFiles(video)
+    }
 
-  if (video.VideoStreamingPlaylists) {
-    await moveHLSFiles(video)
-  }
+    if (video.VideoStreamingPlaylists) {
+      await moveHLSFiles(video)
+    }
+
+    const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
+    if (pendingMove === 0) {
+      logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
+      await doAfterLastJob(video, payload.isNewVideo)
+    }
+  } catch (err) {
+    logger.error('Cannot move video %s to object storage.', video.url, { err })
 
-  const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
-  if (pendingMove === 0) {
-    logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
-    await doAfterLastJob(video, payload.isNewVideo)
+    await moveToFailedMoveToObjectStorageState(video)
+    await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
   }
 
   return payload.videoUUID
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index e420991cd..97ff540ed 100644
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
 import { sequelizeTypescript } from '@server/initializers/database'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
-import { MVideoFullLight, MVideoUUID } from '@server/types/models'
+import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models'
 import { VideoState } from '@shared/models'
 import { federateVideoIfNeeded } from './activitypub/videos'
 import { Notifier } from './notifier'
@@ -79,18 +79,25 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
   }
 }
 
-function moveToFailedTranscodingState (video: MVideoFullLight) {
+function moveToFailedTranscodingState (video: MVideo) {
   if (video.state === VideoState.TRANSCODING_FAILED) return
 
   return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
 }
 
+function moveToFailedMoveToObjectStorageState (video: MVideo) {
+  if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return
+
+  return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   buildNextVideoState,
   moveToExternalStorageState,
   moveToFailedTranscodingState,
+  moveToFailedMoveToObjectStorageState,
   moveToNextState
 }