server/lib/job-queue/handlers/video-import.ts

   1 import * as Bull from 'bull'
   2 import { logger } from '../../../helpers/logger'
   3 import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
   4 import { VideoImportModel } from '../../../models/video/video-import'
   5 import { VideoImportState } from '../../../../shared/models/videos'
   6 import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
   7 import { extname } from 'path'
   8 import { VideoFileModel } from '../../../models/video/video-file'
   9 import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
  10 import { VideoState } from '../../../../shared'
  11 import { federateVideoIfNeeded } from '../../activitypub'
  12 import { VideoModel } from '../../../models/video/video'
  13 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
  14 import { getSecureTorrentName } from '../../../helpers/utils'
  15 import { move, remove, stat } from 'fs-extra'
  16 import { Notifier } from '../../notifier'
  17 import { CONFIG } from '../../../initializers/config'
  18 import { sequelizeTypescript } from '../../../initializers/database'
  19 import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumbnail'
  20 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
  21 import { MThumbnail } from '../../../typings/models/video/thumbnail'
  22 import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
  23 import { getVideoFilePath } from '@server/lib/video-paths'
  24 import { addOptimizeOrMergeAudioJob } from '@server/lib/videos'
  25
  26 type VideoImportYoutubeDLPayload = {
  27   type: 'youtube-dl'
  28   videoImportId: number
  29
  30   thumbnailUrl: string
  31   downloadThumbnail: boolean
  32   downloadPreview: boolean
  33
  34   fileExt?: string
  35 }
  36
  37 type VideoImportTorrentPayload = {
  38   type: 'magnet-uri' | 'torrent-file'
  39   videoImportId: number
  40 }
  41
  42 export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
  43
  44 async function processVideoImport (job: Bull.Job) {
  45   const payload = job.data as VideoImportPayload
  46
  47   if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
  48   if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, payload)
  49 }
  50
  51 // ---------------------------------------------------------------------------
  52
  53 export {
  54   processVideoImport
  55 }
  56
  57 // ---------------------------------------------------------------------------
  58
  59 async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
  60   logger.info('Processing torrent video import in job %d.', job.id)
  61
  62   const videoImport = await getVideoImportOrDie(payload.videoImportId)
  63
  64   const options = {
  65     videoImportId: payload.videoImportId,
  66
  67     downloadThumbnail: false,
  68     downloadPreview: false,
  69
  70     generateThumbnail: true,
  71     generatePreview: true
  72   }
  73   const target = {
  74     torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
  75     magnetUri: videoImport.magnetUri
  76   }
  77   return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options)
  78 }
  79
  80 async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
  81   logger.info('Processing youtubeDL video import in job %d.', job.id)
  82
  83   const videoImport = await getVideoImportOrDie(payload.videoImportId)
  84   const options = {
  85     videoImportId: videoImport.id,
  86
  87     downloadThumbnail: payload.downloadThumbnail,
  88     downloadPreview: payload.downloadPreview,
  89     thumbnailUrl: payload.thumbnailUrl,
  90
  91     generateThumbnail: false,
  92     generatePreview: false
  93   }
  94
  95   return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), videoImport, options)
  96 }
  97
  98 async function getVideoImportOrDie (videoImportId: number) {
  99   const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
 100   if (!videoImport || !videoImport.Video) {
 101     throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
 102   }
 103
 104   return videoImport
 105 }
 106
 107 type ProcessFileOptions = {
 108   videoImportId: number
 109
 110   downloadThumbnail: boolean
 111   downloadPreview: boolean
 112   thumbnailUrl?: string
 113
 114   generateThumbnail: boolean
 115   generatePreview: boolean
 116 }
 117 async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
 118   let tempVideoPath: string
 119   let videoDestFile: string
 120   let videoFile: VideoFileModel
 121
 122   try {
 123     // Download video from youtubeDL
 124     tempVideoPath = await downloader()
 125
 126     // Get information about this video
 127     const stats = await stat(tempVideoPath)
 128     const isAble = await videoImport.User.isAbleToUploadVideo({ size: stats.size })
 129     if (isAble === false) {
 130       throw new Error('The user video quota is exceeded with this video to import.')
 131     }
 132
 133     const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
 134     const fps = await getVideoFileFPS(tempVideoPath)
 135     const duration = await getDurationFromVideoFile(tempVideoPath)
 136
 137     // Create video file object in database
 138     const videoFileData = {
 139       extname: extname(tempVideoPath),
 140       resolution: videoFileResolution,
 141       size: stats.size,
 142       fps,
 143       videoId: videoImport.videoId
 144     }
 145     videoFile = new VideoFileModel(videoFileData)
 146
 147     const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
 148     // To clean files if the import fails
 149     const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
 150
 151     // Move file
 152     videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile)
 153     await move(tempVideoPath, videoDestFile)
 154     tempVideoPath = null // This path is not used anymore
 155
 156     // Process thumbnail
 157     let thumbnailModel: MThumbnail
 158     if (options.downloadThumbnail && options.thumbnailUrl) {
 159       try {
 160         thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.MINIATURE)
 161       } catch (err) {
 162         logger.warn('Cannot generate video thumbnail %s for %s.', options.thumbnailUrl, videoImportWithFiles.Video.url, { err })
 163       }
 164     }
 165
 166     if (!thumbnailModel && (options.generateThumbnail || options.downloadThumbnail)) {
 167       thumbnailModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.MINIATURE)
 168     }
 169
 170     // Process preview
 171     let previewModel: MThumbnail
 172     if (options.downloadPreview && options.thumbnailUrl) {
 173       try {
 174         previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.PREVIEW)
 175       } catch (err) {
 176         logger.warn('Cannot generate video preview %s for %s.', options.thumbnailUrl, videoImportWithFiles.Video.url, { err })
 177       }
 178     }
 179
 180     if (!previewModel && (options.generatePreview || options.downloadPreview)) {
 181       previewModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.PREVIEW)
 182     }
 183
 184     // Create torrent
 185     await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
 186
 187     const { videoImportUpdated, video } = await sequelizeTypescript.transaction(async t => {
 188       const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo
 189
 190       // Refresh video
 191       const video = await VideoModel.load(videoImportToUpdate.videoId, t)
 192       if (!video) throw new Error('Video linked to import ' + videoImportToUpdate.videoId + ' does not exist anymore.')
 193
 194       const videoFileCreated = await videoFile.save({ transaction: t })
 195       videoImportToUpdate.Video = Object.assign(video, { VideoFiles: [ videoFileCreated ] })
 196
 197       // Update video DB object
 198       video.duration = duration
 199       video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
 200       await video.save({ transaction: t })
 201
 202       if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
 203       if (previewModel) await video.addAndSaveThumbnail(previewModel, t)
 204
 205       // Now we can federate the video (reload from database, we need more attributes)
 206       const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
 207       await federateVideoIfNeeded(videoForFederation, true, t)
 208
 209       // Update video import object
 210       videoImportToUpdate.state = VideoImportState.SUCCESS
 211       const videoImportUpdated = await videoImportToUpdate.save({ transaction: t }) as MVideoImportVideo
 212       videoImportUpdated.Video = video
 213
 214       logger.info('Video %s imported.', video.uuid)
 215
 216       return { videoImportUpdated, video: videoForFederation }
 217     })
 218
 219     Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true)
 220
 221     if (video.isBlacklisted()) {
 222       const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
 223
 224       Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
 225     } else {
 226       Notifier.Instance.notifyOnNewVideoIfNeeded(video)
 227     }
 228
 229     // Create transcoding jobs?
 230     if (video.state === VideoState.TO_TRANSCODE) {
 231       await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile)
 232     }
 233
 234   } catch (err) {
 235     try {
 236       if (tempVideoPath) await remove(tempVideoPath)
 237     } catch (errUnlink) {
 238       logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
 239     }
 240
 241     videoImport.error = err.message
 242     videoImport.state = VideoImportState.FAILED
 243     await videoImport.save()
 244
 245     Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false)
 246
 247     throw err
 248   }
 249 }