1 import { Job } from 'bull'
2 import { remove } from 'fs-extra'
3 import { join } from 'path'
4 import { logger } from '@server/helpers/logger'
5 import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6 import { CONFIG } from '@server/initializers/config'
7 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
8 import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
9 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
10 import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
11 import { VideoModel } from '@server/models/video/video'
12 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
13 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
14 import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared'
16 export async function processMoveToObjectStorage (job: Job) {
17 const payload = job.data as MoveObjectStoragePayload
18 logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
20 const video = await VideoModel.loadWithFiles(payload.videoUUID)
21 // No video, maybe deleted?
23 logger.info('Can\'t process job %d, video does not exist.', job.id)
28 if (video.VideoFiles) {
29 await moveWebTorrentFiles(video)
32 if (video.VideoStreamingPlaylists) {
33 await moveHLSFiles(video)
36 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
37 if (pendingMove === 0) {
38 logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
39 await doAfterLastJob(video, payload.isNewVideo)
42 logger.error('Cannot move video %s to object storage.', video.url, { err })
44 await moveToFailedMoveToObjectStorageState(video)
45 await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
48 return payload.videoUUID
51 // ---------------------------------------------------------------------------
53 async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
54 for (const file of video.VideoFiles) {
55 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
57 const fileUrl = await storeWebTorrentFile(file.filename)
59 const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename)
60 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
64 async function moveHLSFiles (video: MVideoWithAllFiles) {
65 for (const playlist of video.VideoStreamingPlaylists) {
66 const playlistWithVideo = playlist.withVideo(video)
68 for (const file of playlist.VideoFiles) {
69 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
71 // Resolution playlist
72 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
73 await storeHLSFile(playlistWithVideo, playlistFilename)
75 // Resolution fragmented file
76 const fileUrl = await storeHLSFile(playlistWithVideo, file.filename)
78 const oldPath = join(getHLSDirectory(video), file.filename)
80 await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
85 async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) {
86 for (const playlist of video.VideoStreamingPlaylists) {
87 if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
89 const playlistWithVideo = playlist.withVideo(video)
92 playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename)
93 // Sha256 segments file
94 playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename)
96 playlist.storage = VideoStorage.OBJECT_STORAGE
98 playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
99 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
101 await playlist.save()
104 // Remove empty hls video directory
105 if (video.VideoStreamingPlaylists) {
106 await remove(getHLSDirectory(video))
109 await moveToNextState(video, isNewVideo)
112 async function onFileMoved (options: {
113 videoOrPlaylist: MVideo | MStreamingPlaylistVideo
118 const { videoOrPlaylist, file, fileUrl, oldPath } = options
120 file.fileUrl = fileUrl
121 file.storage = VideoStorage.OBJECT_STORAGE
123 await updateTorrentMetadata(videoOrPlaylist, file)
126 logger.debug('Removing %s because it\'s now on object storage', oldPath)
127 await remove(oldPath)