aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/job-queue/handlers/move-to-object-storage.ts
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib/job-queue/handlers/move-to-object-storage.ts')
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts159
1 files changed, 0 insertions, 159 deletions
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
deleted file mode 100644
index 9a99b6722..000000000
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ /dev/null
@@ -1,159 +0,0 @@
1import { Job } from 'bullmq'
2import { remove } from 'fs-extra'
3import { join } from 'path'
4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
7import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage'
8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
9import { VideoPathManager } from '@server/lib/video-path-manager'
10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video'
12import { VideoJobInfoModel } from '@server/models/video/video-job-info'
13import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
14import { MoveObjectStoragePayload, VideoState, VideoStorage } from '@shared/models'
15
16const lTagsBase = loggerTagsFactory('move-object-storage')
17
18export async function processMoveToObjectStorage (job: Job) {
19 const payload = job.data as MoveObjectStoragePayload
20 logger.info('Moving video %s in job %s.', payload.videoUUID, job.id)
21
22 const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
23
24 const video = await VideoModel.loadWithFiles(payload.videoUUID)
25 // No video, maybe deleted?
26 if (!video) {
27 logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
28 fileMutexReleaser()
29 return undefined
30 }
31
32 const lTags = lTagsBase(video.uuid, video.url)
33
34 try {
35 if (video.VideoFiles) {
36 logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
37
38 await moveWebVideoFiles(video)
39 }
40
41 if (video.VideoStreamingPlaylists) {
42 logger.debug('Moving HLS playlist of %s.', video.uuid)
43
44 await moveHLSFiles(video)
45 }
46
47 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
48 if (pendingMove === 0) {
49 logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags)
50
51 await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
52 }
53 } catch (err) {
54 await onMoveToObjectStorageFailure(job, err)
55
56 throw err
57 } finally {
58 fileMutexReleaser()
59 }
60
61 return payload.videoUUID
62}
63
64export async function onMoveToObjectStorageFailure (job: Job, err: any) {
65 const payload = job.data as MoveObjectStoragePayload
66
67 const video = await VideoModel.loadWithFiles(payload.videoUUID)
68 if (!video) return
69
70 logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) })
71
72 await moveToFailedMoveToObjectStorageState(video)
73 await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
74}
75
76// ---------------------------------------------------------------------------
77
78async function moveWebVideoFiles (video: MVideoWithAllFiles) {
79 for (const file of video.VideoFiles) {
80 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
81
82 const fileUrl = await storeWebVideoFile(video, file)
83
84 const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
85 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
86 }
87}
88
89async function moveHLSFiles (video: MVideoWithAllFiles) {
90 for (const playlist of video.VideoStreamingPlaylists) {
91 const playlistWithVideo = playlist.withVideo(video)
92
93 for (const file of playlist.VideoFiles) {
94 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
95
96 // Resolution playlist
97 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
98 await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
99
100 // Resolution fragmented file
101 const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
102
103 const oldPath = join(getHLSDirectory(video), file.filename)
104
105 await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
106 }
107 }
108}
109
110async function doAfterLastJob (options: {
111 video: MVideoWithAllFiles
112 previousVideoState: VideoState
113 isNewVideo: boolean
114}) {
115 const { video, previousVideoState, isNewVideo } = options
116
117 for (const playlist of video.VideoStreamingPlaylists) {
118 if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
119
120 const playlistWithVideo = playlist.withVideo(video)
121
122 // Master playlist
123 playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
124 // Sha256 segments file
125 playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
126
127 playlist.storage = VideoStorage.OBJECT_STORAGE
128
129 playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
130 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
131
132 await playlist.save()
133 }
134
135 // Remove empty hls video directory
136 if (video.VideoStreamingPlaylists) {
137 await remove(getHLSDirectory(video))
138 }
139
140 await moveToNextState({ video, previousVideoState, isNewVideo })
141}
142
143async function onFileMoved (options: {
144 videoOrPlaylist: MVideo | MStreamingPlaylistVideo
145 file: MVideoFile
146 fileUrl: string
147 oldPath: string
148}) {
149 const { videoOrPlaylist, file, fileUrl, oldPath } = options
150
151 file.fileUrl = fileUrl
152 file.storage = VideoStorage.OBJECT_STORAGE
153
154 await updateTorrentMetadata(videoOrPlaylist, file)
155 await file.save()
156
157 logger.debug('Removing %s because it\'s now on object storage', oldPath)
158 await remove(oldPath)
159}
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-21 09:43:50 +0000