]>
Commit | Line | Data |
---|---|---|
5a921e7b | 1 | import { Job } from 'bullmq' |
0305db28 JB |
2 | import { remove } from 'fs-extra' |
3 | import { join } from 'path' | |
a2caee9f | 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
9b293cd6 | 5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' |
0305db28 | 6 | import { CONFIG } from '@server/initializers/config' |
fb72d2e1 | 7 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' |
cfd57d2c | 8 | import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage' |
0305db28 | 9 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' |
dbd9fb44 | 10 | import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' |
0305db28 JB |
11 | import { VideoModel } from '@server/models/video/video' |
12 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | |
13 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' | |
1808a1f8 | 14 | import { MoveObjectStoragePayload, VideoState, VideoStorage } from '@shared/models' |
0305db28 | 15 | |
a2caee9f C |
16 | const lTagsBase = loggerTagsFactory('move-object-storage') |
17 | ||
41fb13c3 | 18 | export async function processMoveToObjectStorage (job: Job) { |
0305db28 | 19 | const payload = job.data as MoveObjectStoragePayload |
bd911b54 | 20 | logger.info('Moving video %s in job %s.', payload.videoUUID, job.id) |
0305db28 JB |
21 | |
22 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | |
23 | // No video, maybe deleted? | |
24 | if (!video) { | |
a2caee9f | 25 | logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID)) |
0305db28 JB |
26 | return undefined |
27 | } | |
28 | ||
a2caee9f C |
29 | const lTags = lTagsBase(video.uuid, video.url) |
30 | ||
dbd9fb44 C |
31 | try { |
32 | if (video.VideoFiles) { | |
a2caee9f C |
33 | logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags) |
34 | ||
dbd9fb44 C |
35 | await moveWebTorrentFiles(video) |
36 | } | |
0305db28 | 37 | |
dbd9fb44 | 38 | if (video.VideoStreamingPlaylists) { |
a2caee9f C |
39 | logger.debug('Moving HLS playlist of %s.', video.uuid) |
40 | ||
dbd9fb44 C |
41 | await moveHLSFiles(video) |
42 | } | |
43 | ||
44 | const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') | |
45 | if (pendingMove === 0) { | |
bd911b54 | 46 | logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags) |
a2caee9f | 47 | |
1808a1f8 | 48 | await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) |
dbd9fb44 C |
49 | } |
50 | } catch (err) { | |
32567717 | 51 | await onMoveToObjectStorageFailure(job, err) |
0305db28 JB |
52 | } |
53 | ||
54 | return payload.videoUUID | |
55 | } | |
56 | ||
32567717 C |
57 | export async function onMoveToObjectStorageFailure (job: Job, err: any) { |
58 | const payload = job.data as MoveObjectStoragePayload | |
59 | ||
60 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | |
61 | if (!video) return | |
62 | ||
63 | logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) }) | |
64 | ||
65 | await moveToFailedMoveToObjectStorageState(video) | |
66 | await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') | |
67 | } | |
68 | ||
0305db28 JB |
69 | // --------------------------------------------------------------------------- |
70 | ||
71 | async function moveWebTorrentFiles (video: MVideoWithAllFiles) { | |
72 | for (const file of video.VideoFiles) { | |
73 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | |
74 | ||
75 | const fileUrl = await storeWebTorrentFile(file.filename) | |
76 | ||
77 | const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) | |
78 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) | |
79 | } | |
80 | } | |
81 | ||
82 | async function moveHLSFiles (video: MVideoWithAllFiles) { | |
83 | for (const playlist of video.VideoStreamingPlaylists) { | |
ad5db104 | 84 | const playlistWithVideo = playlist.withVideo(video) |
0305db28 JB |
85 | |
86 | for (const file of playlist.VideoFiles) { | |
87 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | |
88 | ||
89 | // Resolution playlist | |
90 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | |
cfd57d2c | 91 | await storeHLSFileFromFilename(playlistWithVideo, playlistFilename) |
0305db28 JB |
92 | |
93 | // Resolution fragmented file | |
cfd57d2c | 94 | const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename) |
0305db28 JB |
95 | |
96 | const oldPath = join(getHLSDirectory(video), file.filename) | |
97 | ||
98 | await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) | |
99 | } | |
100 | } | |
101 | } | |
102 | ||
1808a1f8 C |
103 | async function doAfterLastJob (options: { |
104 | video: MVideoWithAllFiles | |
105 | previousVideoState: VideoState | |
106 | isNewVideo: boolean | |
107 | }) { | |
108 | const { video, previousVideoState, isNewVideo } = options | |
109 | ||
0305db28 JB |
110 | for (const playlist of video.VideoStreamingPlaylists) { |
111 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue | |
112 | ||
ad5db104 C |
113 | const playlistWithVideo = playlist.withVideo(video) |
114 | ||
0305db28 | 115 | // Master playlist |
cfd57d2c | 116 | playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename) |
0305db28 | 117 | // Sha256 segments file |
cfd57d2c | 118 | playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename) |
0305db28 JB |
119 | |
120 | playlist.storage = VideoStorage.OBJECT_STORAGE | |
121 | ||
fb72d2e1 C |
122 | playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles) |
123 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION | |
124 | ||
0305db28 JB |
125 | await playlist.save() |
126 | } | |
127 | ||
128 | // Remove empty hls video directory | |
129 | if (video.VideoStreamingPlaylists) { | |
130 | await remove(getHLSDirectory(video)) | |
131 | } | |
132 | ||
1808a1f8 | 133 | await moveToNextState({ video, previousVideoState, isNewVideo }) |
0305db28 JB |
134 | } |
135 | ||
136 | async function onFileMoved (options: { | |
137 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo | |
138 | file: MVideoFile | |
139 | fileUrl: string | |
140 | oldPath: string | |
141 | }) { | |
142 | const { videoOrPlaylist, file, fileUrl, oldPath } = options | |
143 | ||
144 | file.fileUrl = fileUrl | |
145 | file.storage = VideoStorage.OBJECT_STORAGE | |
146 | ||
9b293cd6 | 147 | await updateTorrentMetadata(videoOrPlaylist, file) |
0305db28 JB |
148 | await file.save() |
149 | ||
150 | logger.debug('Removing %s because it\'s now on object storage', oldPath) | |
151 | await remove(oldPath) | |
152 | } |