Don't stuck state when move transcoding job failed

9 files changed, 64 insertions, 16 deletions

diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
index 6e5d0bcad..0c4d46714 100644
--- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
+++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
@@ -2,6 +2,10 @@
   Transcoding failed, this video may not work properly.
 </div>
 
+<div i18n class="alert alert-warning" *ngIf="isVideoMoveToObjectStorageFailed()">
+  Move to external storage failed, this video may not work properly.
+</div>
+
 <div i18n class="alert alert-warning" *ngIf="isVideoToImport()">
   The video is being imported, it will be available when the import is finished.
 </div>
diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
index addea53c0..a3d3fa6fb 100644
--- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
+++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
@@ -18,6 +18,10 @@ export class VideoAlertComponent {
     return this.video && this.video.state.id === VideoState.TRANSCODING_FAILED
   }
 
+  isVideoMoveToObjectStorageFailed () {
+    return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED
+  }
+
   isVideoToImport () {
     return this.video && this.video.state.id === VideoState.TO_IMPORT
   }
diff --git a/client/src/app/shared/shared-video-miniature/video-miniature.component.ts b/client/src/app/shared/shared-video-miniature/video-miniature.component.ts
index f387c38c2..847e401ed 100644
--- a/client/src/app/shared/shared-video-miniature/video-miniature.component.ts
+++ b/client/src/app/shared/shared-video-miniature/video-miniature.component.ts
@@ -179,6 +179,10 @@ export class VideoMiniatureComponent implements OnInit {
       return $localize`Transcoding failed`
     }
 
+    if (video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) {
+      return $localize`Move to external storage failed`
+    }
+
     if (video.state.id === VideoState.TO_TRANSCODE && video.waitTranscoding === true) {
       return $localize`Waiting transcoding`
     }
diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts
index 699487f72..90c84b1d7 100644
--- a/scripts/create-move-video-storage-job.ts
+++ b/scripts/create-move-video-storage-job.ts
@@ -4,7 +4,7 @@ registerTSPaths()
 import { program } from 'commander'
 import { VideoModel } from '@server/models/video/video'
 import { initDatabaseModels } from '@server/initializers/database'
-import { VideoStorage } from '@shared/models'
+import { VideoState, VideoStorage } from '@shared/models'
 import { moveToExternalStorageState } from '@server/lib/video-state'
 import { JobQueue } from '@server/lib/job-queue'
 import { CONFIG } from '@server/initializers/config'
@@ -62,6 +62,11 @@ async function run () {
       process.exit(-1)
     }
 
+    if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
+      console.error('This video is already being moved to external storage')
+      process.exit(-1)
+    }
+
     ids.push(video.id)
   } else {
     ids = await VideoModel.listLocalIds()
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 57f7af789..7816561fd 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -427,7 +427,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = {
   [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
   [VideoState.LIVE_ENDED]: 'Livestream ended',
   [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage',
-  [VideoState.TRANSCODING_FAILED]: 'Transcoding failed'
+  [VideoState.TRANSCODING_FAILED]: 'Transcoding failed',
+  [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed'
 }
 
 const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index b5eea0184..d9c415b2d 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
 import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
-import { moveToNextState } from '@server/lib/video-state'
+import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
@@ -24,18 +24,25 @@ export async function processMoveToObjectStorage (job: Job) {
     return undefined
   }
 
-  if (video.VideoFiles) {
-    await moveWebTorrentFiles(video)
-  }
+  try {
+    if (video.VideoFiles) {
+      await moveWebTorrentFiles(video)
+    }
 
-  if (video.VideoStreamingPlaylists) {
-    await moveHLSFiles(video)
-  }
+    if (video.VideoStreamingPlaylists) {
+      await moveHLSFiles(video)
+    }
+
+    const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
+    if (pendingMove === 0) {
+      logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
+      await doAfterLastJob(video, payload.isNewVideo)
+    }
+  } catch (err) {
+    logger.error('Cannot move video %s to object storage.', video.url, { err })
 
-  const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
-  if (pendingMove === 0) {
-    logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
-    await doAfterLastJob(video, payload.isNewVideo)
+    await moveToFailedMoveToObjectStorageState(video)
+    await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
   }
 
   return payload.videoUUID
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index e420991cd..97ff540ed 100644
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
 import { sequelizeTypescript } from '@server/initializers/database'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
-import { MVideoFullLight, MVideoUUID } from '@server/types/models'
+import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models'
 import { VideoState } from '@shared/models'
 import { federateVideoIfNeeded } from './activitypub/videos'
 import { Notifier } from './notifier'
@@ -79,18 +79,25 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
   }
 }
 
-function moveToFailedTranscodingState (video: MVideoFullLight) {
+function moveToFailedTranscodingState (video: MVideo) {
   if (video.state === VideoState.TRANSCODING_FAILED) return
 
   return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
 }
 
+function moveToFailedMoveToObjectStorageState (video: MVideo) {
+  if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return
+
+  return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   buildNextVideoState,
   moveToExternalStorageState,
   moveToFailedTranscodingState,
+  moveToFailedMoveToObjectStorageState,
   moveToNextState
 }
 
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
index 6a67a214c..7497addf1 100644
--- a/server/models/video/video-job-info.ts
+++ b/server/models/video/video-job-info.ts
@@ -99,4 +99,19 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
 
     return pendingMove
   }
+
+  static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> {
+    const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, bind: { videoUUID } }
+
+    await VideoJobInfoModel.sequelize.query(`
+    UPDATE
+      "videoJobInfo"
+    SET
+      "${column}" = 0,
+      "updatedAt" = NOW()
+    FROM "video"
+    WHERE
+      "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID
+    `, options)
+  }
 }
diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts
index 6112b6e16..09268d2ff 100644
--- a/shared/models/videos/video-state.enum.ts
+++ b/shared/models/videos/video-state.enum.ts
@@ -5,5 +5,6 @@ export const enum VideoState {
   WAITING_FOR_LIVE = 4,
   LIVE_ENDED = 5,
   TO_MOVE_TO_EXTERNAL_STORAGE = 6,
-  TRANSCODING_FAILED = 7
+  TRANSCODING_FAILED = 7,
+  TO_MOVE_TO_EXTERNAL_STORAGE_FAILED = 8
 }