Don't stuck state when move transcoding job failed

diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
index 6e5d0bcad6ce204d37562857388ef1ddf40459b1..0c4d46714b543bee60f7cb79219b7d8d11cf30af 100644 (file)
--- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
+++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
@@ -2,6 +2,10 @@
   Transcoding failed, this video may not work properly.
 </div>
 
+<div i18n class="alert alert-warning" *ngIf="isVideoMoveToObjectStorageFailed()">
+  Move to external storage failed, this video may not work properly.
+</div>
+
 <div i18n class="alert alert-warning" *ngIf="isVideoToImport()">
   The video is being imported, it will be available when the import is finished.
 </div>

diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
index addea53c0061ca3dd1b799d865f29e6fe7d8b621..a3d3fa6fbf2212e8011ebd825988c283b93c7041 100644 (file)
--- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
+++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
@@ -18,6 +18,10 @@ export class VideoAlertComponent {
     return this.video && this.video.state.id === VideoState.TRANSCODING_FAILED
   }
 
+  isVideoMoveToObjectStorageFailed () {
+    return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED
+  }
+
   isVideoToImport () {
     return this.video && this.video.state.id === VideoState.TO_IMPORT
   }

diff --git a/client/src/app/shared/shared-video-miniature/video-miniature.component.ts b/client/src/app/shared/shared-video-miniature/video-miniature.component.ts
index f387c38c2d163dd61cb8ecf2d853fe2c0baabc31..847e401ed1837c8a325c83df209a4fd7c53119f4 100644 (file)
--- a/client/src/app/shared/shared-video-miniature/video-miniature.component.ts
+++ b/client/src/app/shared/shared-video-miniature/video-miniature.component.ts
@@ -179,6 +179,10 @@ export class VideoMiniatureComponent implements OnInit {
       return $localize`Transcoding failed`
     }
 
+    if (video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) {
+      return $localize`Move to external storage failed`
+    }
+
     if (video.state.id === VideoState.TO_TRANSCODE && video.waitTranscoding === true) {
       return $localize`Waiting transcoding`
     }

diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts
index 699487f72e7b2a876520e59bb7364ed52356a4f8..90c84b1d75ebc29668135213de1894b1f5100fde 100644 (file)
--- a/scripts/create-move-video-storage-job.ts
+++ b/scripts/create-move-video-storage-job.ts
@@ -4,7 +4,7 @@ registerTSPaths()
 import { program } from 'commander'
 import { VideoModel } from '@server/models/video/video'
 import { initDatabaseModels } from '@server/initializers/database'
-import { VideoStorage } from '@shared/models'
+import { VideoState, VideoStorage } from '@shared/models'
 import { moveToExternalStorageState } from '@server/lib/video-state'
 import { JobQueue } from '@server/lib/job-queue'
 import { CONFIG } from '@server/initializers/config'
@@ -62,6 +62,11 @@ async function run () {
       process.exit(-1)
     }
 
+    if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
+      console.error('This video is already being moved to external storage')
+      process.exit(-1)
+    }
+
     ids.push(video.id)
   } else {
     ids = await VideoModel.listLocalIds()

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 57f7af789ca16969de4b63899222c8960f5070a5..7816561fd08ee1eb4060e85b037e3aac71e330a8 100644 (file)
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -427,7 +427,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = {
   [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
   [VideoState.LIVE_ENDED]: 'Livestream ended',
   [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage',
-  [VideoState.TRANSCODING_FAILED]: 'Transcoding failed'
+  [VideoState.TRANSCODING_FAILED]: 'Transcoding failed',
+  [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed'
 }
 
 const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {

diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index b5eea018448c5e7cf6e3c20a57e67ddbfbcb4778..d9c415b2ddcf4375cf079a43631bbfbebc463efd 100644 (file)
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
 import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
-import { moveToNextState } from '@server/lib/video-state'
+import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
@@ -24,18 +24,25 @@ export async function processMoveToObjectStorage (job: Job) {
     return undefined
   }
 
-  if (video.VideoFiles) {
-    await moveWebTorrentFiles(video)
-  }
+  try {
+    if (video.VideoFiles) {
+      await moveWebTorrentFiles(video)
+    }
 
-  if (video.VideoStreamingPlaylists) {
-    await moveHLSFiles(video)
-  }
+    if (video.VideoStreamingPlaylists) {
+      await moveHLSFiles(video)
+    }
+
+    const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
+    if (pendingMove === 0) {
+      logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
+      await doAfterLastJob(video, payload.isNewVideo)
+    }
+  } catch (err) {
+    logger.error('Cannot move video %s to object storage.', video.url, { err })
 
-  const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
-  if (pendingMove === 0) {
-    logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
-    await doAfterLastJob(video, payload.isNewVideo)
+    await moveToFailedMoveToObjectStorageState(video)
+    await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
   }
 
   return payload.videoUUID

diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index e420991cd34f44a5b945f986f1b4c42b68abc589..97ff540edce61f9db211a74f914dc3d938b1cdda 100644 (file)
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
 import { sequelizeTypescript } from '@server/initializers/database'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
-import { MVideoFullLight, MVideoUUID } from '@server/types/models'
+import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models'
 import { VideoState } from '@shared/models'
 import { federateVideoIfNeeded } from './activitypub/videos'
 import { Notifier } from './notifier'
@@ -79,18 +79,25 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
   }
 }
 
-function moveToFailedTranscodingState (video: MVideoFullLight) {
+function moveToFailedTranscodingState (video: MVideo) {
   if (video.state === VideoState.TRANSCODING_FAILED) return
 
   return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
 }
 
+function moveToFailedMoveToObjectStorageState (video: MVideo) {
+  if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return
+
+  return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   buildNextVideoState,
   moveToExternalStorageState,
   moveToFailedTranscodingState,
+  moveToFailedMoveToObjectStorageState,
   moveToNextState
 }
 

diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
index 6a67a214c13802d823d5233ba7cb0bc31a2d43f0..7497addf119523ac07b8904f42bef75a754788c1 100644 (file)
--- a/server/models/video/video-job-info.ts
+++ b/server/models/video/video-job-info.ts
@@ -99,4 +99,19 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
 
     return pendingMove
   }
+
+  static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> {
+    const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, bind: { videoUUID } }
+
+    await VideoJobInfoModel.sequelize.query(`
+    UPDATE
+      "videoJobInfo"
+    SET
+      "${column}" = 0,
+      "updatedAt" = NOW()
+    FROM "video"
+    WHERE
+      "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID
+    `, options)
+  }
 }

diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts
index 6112b6e16c70f25db82049ee3d00cba71c4336ba..09268d2ff13c005765480ae98282eb31844d4090 100644 (file)
--- a/shared/models/videos/video-state.enum.ts
+++ b/shared/models/videos/video-state.enum.ts
@@ -5,5 +5,6 @@ export const enum VideoState {
   WAITING_FOR_LIVE = 4,
   LIVE_ENDED = 5,
   TO_MOVE_TO_EXTERNAL_STORAGE = 6,
-  TRANSCODING_FAILED = 7
+  TRANSCODING_FAILED = 7,
+  TO_MOVE_TO_EXTERNAL_STORAGE_FAILED = 8
 }