server/lib/job-queue/handlers/move-to-object-storage.ts

   1 import { Job } from 'bullmq'
   2 import { remove } from 'fs-extra'
   3 import { join } from 'path'
   4 import { logger, loggerTagsFactory } from '@server/helpers/logger'
   5 import { updateTorrentMetadata } from '@server/helpers/webtorrent'
   6 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
   7 import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage'
   8 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
   9 import { VideoPathManager } from '@server/lib/video-path-manager'
  10 import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
  11 import { VideoModel } from '@server/models/video/video'
  12 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
  13 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
  14 import { MoveObjectStoragePayload, VideoState, VideoStorage } from '@shared/models'
  15
  16 const lTagsBase = loggerTagsFactory('move-object-storage')
  17
  18 export async function processMoveToObjectStorage (job: Job) {
  19   const payload = job.data as MoveObjectStoragePayload
  20   logger.info('Moving video %s in job %s.', payload.videoUUID, job.id)
  21
  22   const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
  23
  24   const video = await VideoModel.loadWithFiles(payload.videoUUID)
  25   // No video, maybe deleted?
  26   if (!video) {
  27     logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
  28     fileMutexReleaser()
  29     return undefined
  30   }
  31
  32   const lTags = lTagsBase(video.uuid, video.url)
  33
  34   try {
  35     if (video.VideoFiles) {
  36       logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags)
  37
  38       await moveWebTorrentFiles(video)
  39     }
  40
  41     if (video.VideoStreamingPlaylists) {
  42       logger.debug('Moving HLS playlist of %s.', video.uuid)
  43
  44       await moveHLSFiles(video)
  45     }
  46
  47     const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
  48     if (pendingMove === 0) {
  49       logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags)
  50
  51       await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
  52     }
  53   } catch (err) {
  54     await onMoveToObjectStorageFailure(job, err)
  55
  56     throw err
  57   } finally {
  58     fileMutexReleaser()
  59   }
  60
  61   return payload.videoUUID
  62 }
  63
  64 export async function onMoveToObjectStorageFailure (job: Job, err: any) {
  65   const payload = job.data as MoveObjectStoragePayload
  66
  67   const video = await VideoModel.loadWithFiles(payload.videoUUID)
  68   if (!video) return
  69
  70   logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) })
  71
  72   await moveToFailedMoveToObjectStorageState(video)
  73   await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
  74 }
  75
  76 // ---------------------------------------------------------------------------
  77
  78 async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
  79   for (const file of video.VideoFiles) {
  80     if (file.storage !== VideoStorage.FILE_SYSTEM) continue
  81
  82     const fileUrl = await storeWebTorrentFile(video, file)
  83
  84     const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
  85     await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
  86   }
  87 }
  88
  89 async function moveHLSFiles (video: MVideoWithAllFiles) {
  90   for (const playlist of video.VideoStreamingPlaylists) {
  91     const playlistWithVideo = playlist.withVideo(video)
  92
  93     for (const file of playlist.VideoFiles) {
  94       if (file.storage !== VideoStorage.FILE_SYSTEM) continue
  95
  96       // Resolution playlist
  97       const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
  98       await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
  99
 100       // Resolution fragmented file
 101       const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
 102
 103       const oldPath = join(getHLSDirectory(video), file.filename)
 104
 105       await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
 106     }
 107   }
 108 }
 109
 110 async function doAfterLastJob (options: {
 111   video: MVideoWithAllFiles
 112   previousVideoState: VideoState
 113   isNewVideo: boolean
 114 }) {
 115   const { video, previousVideoState, isNewVideo } = options
 116
 117   for (const playlist of video.VideoStreamingPlaylists) {
 118     if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
 119
 120     const playlistWithVideo = playlist.withVideo(video)
 121
 122     // Master playlist
 123     playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
 124     // Sha256 segments file
 125     playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
 126
 127     playlist.storage = VideoStorage.OBJECT_STORAGE
 128
 129     playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
 130     playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
 131
 132     await playlist.save()
 133   }
 134
 135   // Remove empty hls video directory
 136   if (video.VideoStreamingPlaylists) {
 137     await remove(getHLSDirectory(video))
 138   }
 139
 140   await moveToNextState({ video, previousVideoState, isNewVideo })
 141 }
 142
 143 async function onFileMoved (options: {
 144   videoOrPlaylist: MVideo | MStreamingPlaylistVideo
 145   file: MVideoFile
 146   fileUrl: string
 147   oldPath: string
 148 }) {
 149   const { videoOrPlaylist, file, fileUrl, oldPath } = options
 150
 151   file.fileUrl = fileUrl
 152   file.storage = VideoStorage.OBJECT_STORAGE
 153
 154   await updateTorrentMetadata(videoOrPlaylist, file)
 155   await file.save()
 156
 157   logger.debug('Removing %s because it\'s now on object storage', oldPath)
 158   await remove(oldPath)
 159 }