server/lib/job-queue/handlers/video-import.ts

   1 import * as Bull from 'bull'
   2 import { logger } from '../../../helpers/logger'
   3 import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
   4 import { VideoImportModel } from '../../../models/video/video-import'
   5 import { VideoImportState } from '../../../../shared/models/videos'
   6 import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
   7 import { extname, join } from 'path'
   8 import { VideoFileModel } from '../../../models/video/video-file'
   9 import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils'
  10 import { CONFIG, sequelizeTypescript } from '../../../initializers'
  11 import { doRequestAndSaveToFile } from '../../../helpers/requests'
  12 import { VideoState } from '../../../../shared'
  13 import { JobQueue } from '../index'
  14 import { federateVideoIfNeeded } from '../../activitypub'
  15 import { VideoModel } from '../../../models/video/video'
  16 import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
  17
  18 type VideoImportYoutubeDLPayload = {
  19   type: 'youtube-dl'
  20   videoImportId: number
  21
  22   thumbnailUrl: string
  23   downloadThumbnail: boolean
  24   downloadPreview: boolean
  25 }
  26
  27 type VideoImportTorrentPayload = {
  28   type: 'magnet-uri'
  29   videoImportId: number
  30 }
  31
  32 export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
  33
  34 async function processVideoImport (job: Bull.Job) {
  35   const payload = job.data as VideoImportPayload
  36
  37   if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
  38   if (payload.type === 'magnet-uri') return processTorrentImport(job, payload)
  39 }
  40
  41 // ---------------------------------------------------------------------------
  42
  43 export {
  44   processVideoImport
  45 }
  46
  47 // ---------------------------------------------------------------------------
  48
  49 async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
  50   logger.info('Processing torrent video import in job %d.', job.id)
  51
  52   const videoImport = await getVideoImportOrDie(payload.videoImportId)
  53   const options = {
  54     videoImportId: payload.videoImportId,
  55
  56     downloadThumbnail: false,
  57     downloadPreview: false,
  58
  59     generateThumbnail: true,
  60     generatePreview: true
  61   }
  62   return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options)
  63 }
  64
  65 async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
  66   logger.info('Processing youtubeDL video import in job %d.', job.id)
  67
  68   const videoImport = await getVideoImportOrDie(payload.videoImportId)
  69   const options = {
  70     videoImportId: videoImport.id,
  71
  72     downloadThumbnail: payload.downloadThumbnail,
  73     downloadPreview: payload.downloadPreview,
  74     thumbnailUrl: payload.thumbnailUrl,
  75
  76     generateThumbnail: false,
  77     generatePreview: false
  78   }
  79
  80   return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options)
  81 }
  82
  83 async function getVideoImportOrDie (videoImportId: number) {
  84   const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
  85   if (!videoImport || !videoImport.Video) {
  86     throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
  87   }
  88
  89   return videoImport
  90 }
  91
  92 type ProcessFileOptions = {
  93   videoImportId: number
  94
  95   downloadThumbnail: boolean
  96   downloadPreview: boolean
  97   thumbnailUrl?: string
  98
  99   generateThumbnail: boolean
 100   generatePreview: boolean
 101 }
 102 async function processFile (downloader: () => Promise<string>, videoImport: VideoImportModel, options: ProcessFileOptions) {
 103   let tempVideoPath: string
 104   let videoDestFile: string
 105   let videoFile: VideoFileModel
 106   try {
 107     // Download video from youtubeDL
 108     tempVideoPath = await downloader()
 109
 110     // Get information about this video
 111     const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
 112     const fps = await getVideoFileFPS(tempVideoPath)
 113     const stats = await statPromise(tempVideoPath)
 114     const duration = await getDurationFromVideoFile(tempVideoPath)
 115
 116     // Create video file object in database
 117     const videoFileData = {
 118       extname: extname(tempVideoPath),
 119       resolution: videoFileResolution,
 120       size: stats.size,
 121       fps,
 122       videoId: videoImport.videoId
 123     }
 124     videoFile = new VideoFileModel(videoFileData)
 125     // Import if the import fails, to clean files
 126     videoImport.Video.VideoFiles = [ videoFile ]
 127
 128     // Move file
 129     videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile))
 130     await renamePromise(tempVideoPath, videoDestFile)
 131     tempVideoPath = null // This path is not used anymore
 132
 133     // Process thumbnail
 134     if (options.downloadThumbnail) {
 135       if (options.thumbnailUrl) {
 136         const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
 137         await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath)
 138       } else {
 139         await videoImport.Video.createThumbnail(videoFile)
 140       }
 141     } else if (options.generateThumbnail) {
 142       await videoImport.Video.createThumbnail(videoFile)
 143     }
 144
 145     // Process preview
 146     if (options.downloadPreview) {
 147       if (options.thumbnailUrl) {
 148         const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
 149         await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath)
 150       } else {
 151         await videoImport.Video.createPreview(videoFile)
 152       }
 153     } else if (options.generatePreview) {
 154       await videoImport.Video.createPreview(videoFile)
 155     }
 156
 157     // Create torrent
 158     await videoImport.Video.createTorrentAndSetInfoHash(videoFile)
 159
 160     const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => {
 161       // Refresh video
 162       const video = await VideoModel.load(videoImport.videoId, t)
 163       if (!video) throw new Error('Video linked to import ' + videoImport.videoId + ' does not exist anymore.')
 164       videoImport.Video = video
 165
 166       const videoFileCreated = await videoFile.save({ transaction: t })
 167       video.VideoFiles = [ videoFileCreated ]
 168
 169       // Update video DB object
 170       video.duration = duration
 171       video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
 172       const videoUpdated = await video.save({ transaction: t })
 173
 174       // Now we can federate the video (reload from database, we need more attributes)
 175       const videoForFederation = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t)
 176       await federateVideoIfNeeded(videoForFederation, true, t)
 177
 178       // Update video import object
 179       videoImport.state = VideoImportState.SUCCESS
 180       const videoImportUpdated = await videoImport.save({ transaction: t })
 181
 182       logger.info('Video %s imported.', videoImport.targetUrl)
 183
 184       videoImportUpdated.Video = videoUpdated
 185       return videoImportUpdated
 186     })
 187
 188     // Create transcoding jobs?
 189     if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) {
 190       // Put uuid because we don't have id auto incremented for now
 191       const dataInput = {
 192         videoUUID: videoImportUpdated.Video.uuid,
 193         isNewVideo: true
 194       }
 195
 196       await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput })
 197     }
 198
 199   } catch (err) {
 200     try {
 201       if (tempVideoPath) await unlinkPromise(tempVideoPath)
 202     } catch (errUnlink) {
 203       logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
 204     }
 205
 206     videoImport.error = err.message
 207     videoImport.state = VideoImportState.FAILED
 208     await videoImport.save()
 209
 210     throw err
 211   }
 212 }