Don't stuck state when move transcoding job failed

4 files changed, 44 insertions, 14 deletions

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 57f7af789..7816561fd 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -427,7 +427,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = {
   [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
   [VideoState.LIVE_ENDED]: 'Livestream ended',
   [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage',
-  [VideoState.TRANSCODING_FAILED]: 'Transcoding failed'
+  [VideoState.TRANSCODING_FAILED]: 'Transcoding failed',
+  [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed'
 }
 
 const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index b5eea0184..d9c415b2d 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
 import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
-import { moveToNextState } from '@server/lib/video-state'
+import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
@@ -24,18 +24,25 @@ export async function processMoveToObjectStorage (job: Job) {
     return undefined
   }
 
-  if (video.VideoFiles) {
-    await moveWebTorrentFiles(video)
-  }
+  try {
+    if (video.VideoFiles) {
+      await moveWebTorrentFiles(video)
+    }
 
-  if (video.VideoStreamingPlaylists) {
-    await moveHLSFiles(video)
-  }
+    if (video.VideoStreamingPlaylists) {
+      await moveHLSFiles(video)
+    }
+
+    const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
+    if (pendingMove === 0) {
+      logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
+      await doAfterLastJob(video, payload.isNewVideo)
+    }
+  } catch (err) {
+    logger.error('Cannot move video %s to object storage.', video.url, { err })
 
-  const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
-  if (pendingMove === 0) {
-    logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
-    await doAfterLastJob(video, payload.isNewVideo)
+    await moveToFailedMoveToObjectStorageState(video)
+    await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
   }
 
   return payload.videoUUID
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index e420991cd..97ff540ed 100644
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
 import { sequelizeTypescript } from '@server/initializers/database'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
-import { MVideoFullLight, MVideoUUID } from '@server/types/models'
+import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models'
 import { VideoState } from '@shared/models'
 import { federateVideoIfNeeded } from './activitypub/videos'
 import { Notifier } from './notifier'
@@ -79,18 +79,25 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
   }
 }
 
-function moveToFailedTranscodingState (video: MVideoFullLight) {
+function moveToFailedTranscodingState (video: MVideo) {
   if (video.state === VideoState.TRANSCODING_FAILED) return
 
   return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
 }
 
+function moveToFailedMoveToObjectStorageState (video: MVideo) {
+  if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return
+
+  return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   buildNextVideoState,
   moveToExternalStorageState,
   moveToFailedTranscodingState,
+  moveToFailedMoveToObjectStorageState,
   moveToNextState
 }
 
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
index 6a67a214c..7497addf1 100644
--- a/server/models/video/video-job-info.ts
+++ b/server/models/video/video-job-info.ts
@@ -99,4 +99,19 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
 
     return pendingMove
   }
+
+  static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> {
+    const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, bind: { videoUUID } }
+
+    await VideoJobInfoModel.sequelize.query(`
+    UPDATE
+      "videoJobInfo"
+    SET
+      "${column}" = 0,
+      "updatedAt" = NOW()
+    FROM "video"
+    WHERE
+      "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID
+    `, options)
+  }
 }