From dbd9fb44ddd880622265097bd7baf4dd71ea0861 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 23 Dec 2021 11:09:31 +0100 Subject: [PATCH] Don't stuck state when move transcoding job failed --- .../information/video-alert.component.html | 4 +++ .../information/video-alert.component.ts | 4 +++ .../video-miniature.component.ts | 4 +++ scripts/create-move-video-storage-job.ts | 7 ++++- server/initializers/constants.ts | 3 +- .../handlers/move-to-object-storage.ts | 29 ++++++++++++------- server/lib/video-state.ts | 11 +++++-- server/models/video/video-job-info.ts | 15 ++++++++++ shared/models/videos/video-state.enum.ts | 3 +- 9 files changed, 64 insertions(+), 16 deletions(-) diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html index 6e5d0bcad..0c4d46714 100644 --- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html +++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html @@ -2,6 +2,10 @@ Transcoding failed, this video may not work properly. +
+ Move to external storage failed, this video may not work properly. +
+
The video is being imported, it will be available when the import is finished.
diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts index addea53c0..a3d3fa6fb 100644 --- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts +++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts @@ -18,6 +18,10 @@ export class VideoAlertComponent { return this.video && this.video.state.id === VideoState.TRANSCODING_FAILED } + isVideoMoveToObjectStorageFailed () { + return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED + } + isVideoToImport () { return this.video && this.video.state.id === VideoState.TO_IMPORT } diff --git a/client/src/app/shared/shared-video-miniature/video-miniature.component.ts b/client/src/app/shared/shared-video-miniature/video-miniature.component.ts index f387c38c2..847e401ed 100644 --- a/client/src/app/shared/shared-video-miniature/video-miniature.component.ts +++ b/client/src/app/shared/shared-video-miniature/video-miniature.component.ts @@ -179,6 +179,10 @@ export class VideoMiniatureComponent implements OnInit { return $localize`Transcoding failed` } + if (video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) { + return $localize`Move to external storage failed` + } + if (video.state.id === VideoState.TO_TRANSCODE && video.waitTranscoding === true) { return $localize`Waiting transcoding` } diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts index 699487f72..90c84b1d7 100644 --- a/scripts/create-move-video-storage-job.ts +++ b/scripts/create-move-video-storage-job.ts @@ -4,7 +4,7 @@ registerTSPaths() import { program } from 'commander' import { VideoModel } from '@server/models/video/video' import { initDatabaseModels } from '@server/initializers/database' -import { VideoStorage } from '@shared/models' +import { VideoState, VideoStorage } from '@shared/models' import { moveToExternalStorageState } from '@server/lib/video-state' import { JobQueue } from '@server/lib/job-queue' import { CONFIG } from '@server/initializers/config' @@ -62,6 +62,11 @@ async function run () { process.exit(-1) } + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + console.error('This video is already being moved to external storage') + process.exit(-1) + } + ids.push(video.id) } else { ids = await VideoModel.listLocalIds() diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 57f7af789..7816561fd 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -427,7 +427,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = { [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', [VideoState.LIVE_ENDED]: 'Livestream ended', [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage', - [VideoState.TRANSCODING_FAILED]: 'Transcoding failed' + [VideoState.TRANSCODING_FAILED]: 'Transcoding failed', + [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed' } const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index b5eea0184..d9c415b2d 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts @@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' -import { moveToNextState } from '@server/lib/video-state' +import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' import { VideoModel } from '@server/models/video/video' import { VideoJobInfoModel } from '@server/models/video/video-job-info' import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' @@ -24,18 +24,25 @@ export async function processMoveToObjectStorage (job: Job) { return undefined } - if (video.VideoFiles) { - await moveWebTorrentFiles(video) - } + try { + if (video.VideoFiles) { + await moveWebTorrentFiles(video) + } - if (video.VideoStreamingPlaylists) { - await moveHLSFiles(video) - } + if (video.VideoStreamingPlaylists) { + await moveHLSFiles(video) + } + + const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') + if (pendingMove === 0) { + logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id) + await doAfterLastJob(video, payload.isNewVideo) + } + } catch (err) { + logger.error('Cannot move video %s to object storage.', video.url, { err }) - const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') - if (pendingMove === 0) { - logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id) - await doAfterLastJob(video, payload.isNewVideo) + await moveToFailedMoveToObjectStorageState(video) + await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') } return payload.videoUUID diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts index e420991cd..97ff540ed 100644 --- a/server/lib/video-state.ts +++ b/server/lib/video-state.ts @@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config' import { sequelizeTypescript } from '@server/initializers/database' import { VideoModel } from '@server/models/video/video' import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideoFullLight, MVideoUUID } from '@server/types/models' +import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models' import { VideoState } from '@shared/models' import { federateVideoIfNeeded } from './activitypub/videos' import { Notifier } from './notifier' @@ -79,18 +79,25 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b } } -function moveToFailedTranscodingState (video: MVideoFullLight) { +function moveToFailedTranscodingState (video: MVideo) { if (video.state === VideoState.TRANSCODING_FAILED) return return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined) } +function moveToFailedMoveToObjectStorageState (video: MVideo) { + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return + + return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined) +} + // --------------------------------------------------------------------------- export { buildNextVideoState, moveToExternalStorageState, moveToFailedTranscodingState, + moveToFailedMoveToObjectStorageState, moveToNextState } diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts index 6a67a214c..7497addf1 100644 --- a/server/models/video/video-job-info.ts +++ b/server/models/video/video-job-info.ts @@ -99,4 +99,19 @@ export class VideoJobInfoModel extends Model { + const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, bind: { videoUUID } } + + await VideoJobInfoModel.sequelize.query(` + UPDATE + "videoJobInfo" + SET + "${column}" = 0, + "updatedAt" = NOW() + FROM "video" + WHERE + "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID + `, options) + } } diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts index 6112b6e16..09268d2ff 100644 --- a/shared/models/videos/video-state.enum.ts +++ b/shared/models/videos/video-state.enum.ts @@ -5,5 +5,6 @@ export const enum VideoState { WAITING_FOR_LIVE = 4, LIVE_ENDED = 5, TO_MOVE_TO_EXTERNAL_STORAGE = 6, - TRANSCODING_FAILED = 7 + TRANSCODING_FAILED = 7, + TO_MOVE_TO_EXTERNAL_STORAGE_FAILED = 8 } -- 2.41.0