server/lib/job-queue/handlers/move-to-object-storage.ts

   1 import { Job } from 'bull'
   2 import { remove } from 'fs-extra'
   3 import { join } from 'path'
   4 import { logger } from '@server/helpers/logger'
   5 import { updateTorrentMetadata } from '@server/helpers/webtorrent'
   6 import { CONFIG } from '@server/initializers/config'
   7 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
   8 import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
   9 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
  10 import { moveToNextState } from '@server/lib/video-state'
  11 import { VideoModel } from '@server/models/video/video'
  12 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
  13 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
  14 import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared'
  15
  16 export async function processMoveToObjectStorage (job: Job) {
  17   const payload = job.data as MoveObjectStoragePayload
  18   logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
  19
  20   const video = await VideoModel.loadWithFiles(payload.videoUUID)
  21   // No video, maybe deleted?
  22   if (!video) {
  23     logger.info('Can\'t process job %d, video does not exist.', job.id)
  24     return undefined
  25   }
  26
  27   if (video.VideoFiles) {
  28     await moveWebTorrentFiles(video)
  29   }
  30
  31   if (video.VideoStreamingPlaylists) {
  32     await moveHLSFiles(video)
  33   }
  34
  35   const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
  36   if (pendingMove === 0) {
  37     logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
  38     await doAfterLastJob(video, payload.isNewVideo)
  39   }
  40
  41   return payload.videoUUID
  42 }
  43
  44 // ---------------------------------------------------------------------------
  45
  46 async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
  47   for (const file of video.VideoFiles) {
  48     if (file.storage !== VideoStorage.FILE_SYSTEM) continue
  49
  50     const fileUrl = await storeWebTorrentFile(file.filename)
  51
  52     const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename)
  53     await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
  54   }
  55 }
  56
  57 async function moveHLSFiles (video: MVideoWithAllFiles) {
  58   for (const playlist of video.VideoStreamingPlaylists) {
  59     const playlistWithVideo = playlist.withVideo(video)
  60
  61     for (const file of playlist.VideoFiles) {
  62       if (file.storage !== VideoStorage.FILE_SYSTEM) continue
  63
  64       // Resolution playlist
  65       const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
  66       await storeHLSFile(playlistWithVideo, playlistFilename)
  67
  68       // Resolution fragmented file
  69       const fileUrl = await storeHLSFile(playlistWithVideo, file.filename)
  70
  71       const oldPath = join(getHLSDirectory(video), file.filename)
  72
  73       await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
  74     }
  75   }
  76 }
  77
  78 async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) {
  79   for (const playlist of video.VideoStreamingPlaylists) {
  80     if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
  81
  82     const playlistWithVideo = playlist.withVideo(video)
  83
  84     // Master playlist
  85     playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename)
  86     // Sha256 segments file
  87     playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename)
  88
  89     playlist.storage = VideoStorage.OBJECT_STORAGE
  90
  91     playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
  92     playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
  93
  94     await playlist.save()
  95   }
  96
  97   // Remove empty hls video directory
  98   if (video.VideoStreamingPlaylists) {
  99     await remove(getHLSDirectory(video))
 100   }
 101
 102   await moveToNextState(video, isNewVideo)
 103 }
 104
 105 async function onFileMoved (options: {
 106   videoOrPlaylist: MVideo | MStreamingPlaylistVideo
 107   file: MVideoFile
 108   fileUrl: string
 109   oldPath: string
 110 }) {
 111   const { videoOrPlaylist, file, fileUrl, oldPath } = options
 112
 113   file.fileUrl = fileUrl
 114   file.storage = VideoStorage.OBJECT_STORAGE
 115
 116   await updateTorrentMetadata(videoOrPlaylist, file)
 117   await file.save()
 118
 119   logger.debug('Removing %s because it\'s now on object storage', oldPath)
 120   await remove(oldPath)
 121 }