server/lib/job-queue/handlers/video-import.ts

   1 import * as Bull from 'bull'
   2 import { logger } from '../../../helpers/logger'
   3 import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
   4 import { VideoImportModel } from '../../../models/video/video-import'
   5 import { VideoImportState } from '../../../../shared/models/videos'
   6 import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
   7 import { extname } from 'path'
   8 import { VideoFileModel } from '../../../models/video/video-file'
   9 import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
  10 import { VideoState } from '../../../../shared'
  11 import { federateVideoIfNeeded } from '../../activitypub'
  12 import { VideoModel } from '../../../models/video/video'
  13 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
  14 import { getSecureTorrentName } from '../../../helpers/utils'
  15 import { move, remove, stat } from 'fs-extra'
  16 import { Notifier } from '../../notifier'
  17 import { CONFIG } from '../../../initializers/config'
  18 import { sequelizeTypescript } from '../../../initializers/database'
  19 import { generateVideoMiniature } from '../../thumbnail'
  20 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
  21 import { MThumbnail } from '../../../typings/models/video/thumbnail'
  22 import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
  23 import { getVideoFilePath } from '@server/lib/video-paths'
  24 import { addOptimizeOrMergeAudioJob } from '@server/lib/videos'
  25
  26 type VideoImportYoutubeDLPayload = {
  27   type: 'youtube-dl'
  28   videoImportId: number
  29
  30   generateThumbnail: boolean
  31   generatePreview: boolean
  32
  33   fileExt?: string
  34 }
  35
  36 type VideoImportTorrentPayload = {
  37   type: 'magnet-uri' | 'torrent-file'
  38   videoImportId: number
  39 }
  40
  41 export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
  42
  43 async function processVideoImport (job: Bull.Job) {
  44   const payload = job.data as VideoImportPayload
  45
  46   if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
  47   if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, payload)
  48 }
  49
  50 // ---------------------------------------------------------------------------
  51
  52 export {
  53   processVideoImport
  54 }
  55
  56 // ---------------------------------------------------------------------------
  57
  58 async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
  59   logger.info('Processing torrent video import in job %d.', job.id)
  60
  61   const videoImport = await getVideoImportOrDie(payload.videoImportId)
  62
  63   const options = {
  64     videoImportId: payload.videoImportId,
  65
  66     generateThumbnail: true,
  67     generatePreview: true
  68   }
  69   const target = {
  70     torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
  71     magnetUri: videoImport.magnetUri
  72   }
  73   return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options)
  74 }
  75
  76 async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
  77   logger.info('Processing youtubeDL video import in job %d.', job.id)
  78
  79   const videoImport = await getVideoImportOrDie(payload.videoImportId)
  80   const options = {
  81     videoImportId: videoImport.id,
  82
  83     generateThumbnail: payload.generateThumbnail,
  84     generatePreview: payload.generatePreview
  85   }
  86
  87   return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), videoImport, options)
  88 }
  89
  90 async function getVideoImportOrDie (videoImportId: number) {
  91   const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
  92   if (!videoImport || !videoImport.Video) {
  93     throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
  94   }
  95
  96   return videoImport
  97 }
  98
  99 type ProcessFileOptions = {
 100   videoImportId: number
 101
 102   generateThumbnail: boolean
 103   generatePreview: boolean
 104 }
 105 async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
 106   let tempVideoPath: string
 107   let videoDestFile: string
 108   let videoFile: VideoFileModel
 109
 110   try {
 111     // Download video from youtubeDL
 112     tempVideoPath = await downloader()
 113
 114     // Get information about this video
 115     const stats = await stat(tempVideoPath)
 116     const isAble = await videoImport.User.isAbleToUploadVideo({ size: stats.size })
 117     if (isAble === false) {
 118       throw new Error('The user video quota is exceeded with this video to import.')
 119     }
 120
 121     const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
 122     const fps = await getVideoFileFPS(tempVideoPath)
 123     const duration = await getDurationFromVideoFile(tempVideoPath)
 124
 125     // Create video file object in database
 126     const videoFileData = {
 127       extname: extname(tempVideoPath),
 128       resolution: videoFileResolution,
 129       size: stats.size,
 130       fps,
 131       videoId: videoImport.videoId
 132     }
 133     videoFile = new VideoFileModel(videoFileData)
 134
 135     const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
 136     // To clean files if the import fails
 137     const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
 138
 139     // Move file
 140     videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile)
 141     await move(tempVideoPath, videoDestFile)
 142     tempVideoPath = null // This path is not used anymore
 143
 144     // Process thumbnail
 145     let thumbnailModel: MThumbnail
 146     if (options.generateThumbnail) {
 147       thumbnailModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.MINIATURE)
 148     }
 149
 150     // Process preview
 151     let previewModel: MThumbnail
 152     if (options.generatePreview) {
 153       previewModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.PREVIEW)
 154     }
 155
 156     // Create torrent
 157     await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
 158
 159     const { videoImportUpdated, video } = await sequelizeTypescript.transaction(async t => {
 160       const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo
 161
 162       // Refresh video
 163       const video = await VideoModel.load(videoImportToUpdate.videoId, t)
 164       if (!video) throw new Error('Video linked to import ' + videoImportToUpdate.videoId + ' does not exist anymore.')
 165
 166       const videoFileCreated = await videoFile.save({ transaction: t })
 167       videoImportToUpdate.Video = Object.assign(video, { VideoFiles: [ videoFileCreated ] })
 168
 169       // Update video DB object
 170       video.duration = duration
 171       video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
 172       await video.save({ transaction: t })
 173
 174       if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
 175       if (previewModel) await video.addAndSaveThumbnail(previewModel, t)
 176
 177       // Now we can federate the video (reload from database, we need more attributes)
 178       const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
 179       await federateVideoIfNeeded(videoForFederation, true, t)
 180
 181       // Update video import object
 182       videoImportToUpdate.state = VideoImportState.SUCCESS
 183       const videoImportUpdated = await videoImportToUpdate.save({ transaction: t }) as MVideoImportVideo
 184       videoImportUpdated.Video = video
 185
 186       logger.info('Video %s imported.', video.uuid)
 187
 188       return { videoImportUpdated, video: videoForFederation }
 189     })
 190
 191     Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true)
 192
 193     if (video.isBlacklisted()) {
 194       const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
 195
 196       Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
 197     } else {
 198       Notifier.Instance.notifyOnNewVideoIfNeeded(video)
 199     }
 200
 201     // Create transcoding jobs?
 202     if (video.state === VideoState.TO_TRANSCODE) {
 203       await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile)
 204     }
 205
 206   } catch (err) {
 207     try {
 208       if (tempVideoPath) await remove(tempVideoPath)
 209     } catch (errUnlink) {
 210       logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
 211     }
 212
 213     videoImport.error = err.message
 214     videoImport.state = VideoImportState.FAILED
 215     await videoImport.save()
 216
 217     Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false)
 218
 219     throw err
 220   }
 221 }