server/lib/video-import.ts

   1 import { remove } from 'fs-extra'
   2 import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
   3 import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
   4 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
   5 import { isResolvingToUnicastOnly } from '@server/helpers/dns'
   6 import { logger } from '@server/helpers/logger'
   7 import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
   8 import { CONFIG } from '@server/initializers/config'
   9 import { sequelizeTypescript } from '@server/initializers/database'
  10 import { Hooks } from '@server/lib/plugins/hooks'
  11 import { ServerConfigManager } from '@server/lib/server-config-manager'
  12 import { setVideoTags } from '@server/lib/video'
  13 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
  14 import { VideoModel } from '@server/models/video/video'
  15 import { VideoCaptionModel } from '@server/models/video/video-caption'
  16 import { VideoImportModel } from '@server/models/video/video-import'
  17 import { FilteredModelAttributes } from '@server/types'
  18 import {
  19   MChannelAccountDefault,
  20   MChannelSync,
  21   MThumbnail,
  22   MUser,
  23   MVideoAccountDefault,
  24   MVideoCaption,
  25   MVideoImportFormattable,
  26   MVideoTag,
  27   MVideoThumbnail,
  28   MVideoWithBlacklistLight
  29 } from '@server/types/models'
  30 import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
  31 import { getLocalVideoActivityPubUrl } from './activitypub/url'
  32 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
  33
  34 class YoutubeDlImportError extends Error {
  35   code: YoutubeDlImportError.CODE
  36   cause?: Error // Property to remove once ES2022 is used
  37   constructor ({ message, code }) {
  38     super(message)
  39     this.code = code
  40   }
  41
  42   static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
  43     const ytDlErr = new this({ message: message ?? err.message, code })
  44     ytDlErr.cause = err
  45     ytDlErr.stack = err.stack // Useless once ES2022 is used
  46     return ytDlErr
  47   }
  48 }
  49
  50 namespace YoutubeDlImportError {
  51   export enum CODE {
  52     FETCH_ERROR,
  53     NOT_ONLY_UNICAST_URL
  54   }
  55 }
  56
  57 // ---------------------------------------------------------------------------
  58
  59 async function insertFromImportIntoDB (parameters: {
  60   video: MVideoThumbnail
  61   thumbnailModel: MThumbnail
  62   previewModel: MThumbnail
  63   videoChannel: MChannelAccountDefault
  64   tags: string[]
  65   videoImportAttributes: FilteredModelAttributes<VideoImportModel>
  66   user: MUser
  67 }): Promise<MVideoImportFormattable> {
  68   const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
  69
  70   const videoImport = await sequelizeTypescript.transaction(async t => {
  71     const sequelizeOptions = { transaction: t }
  72
  73     // Save video object in database
  74     const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
  75     videoCreated.VideoChannel = videoChannel
  76
  77     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
  78     if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
  79
  80     await autoBlacklistVideoIfNeeded({
  81       video: videoCreated,
  82       user,
  83       notify: false,
  84       isRemote: false,
  85       isNew: true,
  86       transaction: t
  87     })
  88
  89     await setVideoTags({ video: videoCreated, tags, transaction: t })
  90
  91     // Create video import object in database
  92     const videoImport = await VideoImportModel.create(
  93       Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
  94       sequelizeOptions
  95     ) as MVideoImportFormattable
  96     videoImport.Video = videoCreated
  97
  98     return videoImport
  99   })
 100
 101   return videoImport
 102 }
 103
 104 async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
 105   channelId: number
 106   importData: YoutubeDLInfo
 107   importDataOverride?: Partial<VideoImportCreate>
 108   importType: 'url' | 'torrent'
 109 }): Promise<MVideoThumbnail> {
 110   let videoData = {
 111     name: importDataOverride?.name || importData.name || 'Unknown name',
 112     remote: false,
 113     category: importDataOverride?.category || importData.category,
 114     licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
 115     language: importDataOverride?.language || importData.language,
 116     commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
 117     downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
 118     waitTranscoding: importDataOverride?.waitTranscoding || false,
 119     state: VideoState.TO_IMPORT,
 120     nsfw: importDataOverride?.nsfw || importData.nsfw || false,
 121     description: importDataOverride?.description || importData.description,
 122     support: importDataOverride?.support || null,
 123     privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
 124     duration: 0, // duration will be set by the import job
 125     channelId,
 126     originallyPublishedAt: importDataOverride?.originallyPublishedAt
 127       ? new Date(importDataOverride?.originallyPublishedAt)
 128       : importData.originallyPublishedAt
 129   }
 130
 131   videoData = await Hooks.wrapObject(
 132     videoData,
 133     importType === 'url'
 134       ? 'filter:api.video.import-url.video-attribute.result'
 135       : 'filter:api.video.import-torrent.video-attribute.result'
 136   )
 137
 138   const video = new VideoModel(videoData)
 139   video.url = getLocalVideoActivityPubUrl(video)
 140
 141   return video
 142 }
 143
 144 async function buildYoutubeDLImport (options: {
 145   targetUrl: string
 146   channel: MChannelAccountDefault
 147   user: MUser
 148   channelSync?: MChannelSync
 149   importDataOverride?: Partial<VideoImportCreate>
 150   thumbnailFilePath?: string
 151   previewFilePath?: string
 152 }) {
 153   const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
 154
 155   const youtubeDL = new YoutubeDLWrapper(
 156     targetUrl,
 157     ServerConfigManager.Instance.getEnabledResolutions('vod'),
 158     CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
 159   )
 160
 161   // Get video infos
 162   let youtubeDLInfo: YoutubeDLInfo
 163   try {
 164     youtubeDLInfo = await youtubeDL.getInfoForDownload()
 165   } catch (err) {
 166     throw YoutubeDlImportError.fromError(
 167       err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
 168     )
 169   }
 170
 171   if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
 172     throw new YoutubeDlImportError({
 173       message: 'Cannot use non unicast IP as targetUrl.',
 174       code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
 175     })
 176   }
 177
 178   const video = await buildVideoFromImport({
 179     channelId: channel.id,
 180     importData: youtubeDLInfo,
 181     importDataOverride,
 182     importType: 'url'
 183   })
 184
 185   const thumbnailModel = await forgeThumbnail({
 186     inputPath: thumbnailFilePath,
 187     downloadUrl: youtubeDLInfo.thumbnailUrl,
 188     video,
 189     type: ThumbnailType.MINIATURE
 190   })
 191
 192   const previewModel = await forgeThumbnail({
 193     inputPath: previewFilePath,
 194     downloadUrl: youtubeDLInfo.thumbnailUrl,
 195     video,
 196     type: ThumbnailType.PREVIEW
 197   })
 198
 199   const videoImport = await insertFromImportIntoDB({
 200     video,
 201     thumbnailModel,
 202     previewModel,
 203     videoChannel: channel,
 204     tags: importDataOverride?.tags || youtubeDLInfo.tags,
 205     user,
 206     videoImportAttributes: {
 207       targetUrl,
 208       state: VideoImportState.PENDING,
 209       userId: user.id
 210     }
 211   })
 212
 213   // Get video subtitles
 214   await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
 215
 216   let fileExt = `.${youtubeDLInfo.ext}`
 217   if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
 218
 219   const payload: VideoImportPayload = {
 220     type: 'youtube-dl' as 'youtube-dl',
 221     videoImportId: videoImport.id,
 222     fileExt,
 223     // If part of a sync process, there is a parent job that will aggregate children results
 224     preventException: !!channelSync
 225   }
 226
 227   return {
 228     videoImport,
 229     job: { type: 'video-import' as 'video-import', payload }
 230   }
 231 }
 232
 233 // ---------------------------------------------------------------------------
 234
 235 export {
 236   buildYoutubeDLImport,
 237   YoutubeDlImportError,
 238   insertFromImportIntoDB,
 239   buildVideoFromImport
 240 }
 241
 242 // ---------------------------------------------------------------------------
 243
 244 async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
 245   inputPath?: string
 246   downloadUrl?: string
 247   video: MVideoThumbnail
 248   type: ThumbnailType
 249 }): Promise<MThumbnail> {
 250   if (inputPath) {
 251     return updateVideoMiniatureFromExisting({
 252       inputPath,
 253       video,
 254       type,
 255       automaticallyGenerated: false
 256     })
 257   } else if (downloadUrl) {
 258     try {
 259       return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
 260     } catch (err) {
 261       logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
 262     }
 263   }
 264   return null
 265 }
 266
 267 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
 268   try {
 269     const subtitles = await youtubeDL.getSubtitles()
 270
 271     logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
 272
 273     for (const subtitle of subtitles) {
 274       if (!await isVTTFileValid(subtitle.path)) {
 275         await remove(subtitle.path)
 276         continue
 277       }
 278
 279       const videoCaption = new VideoCaptionModel({
 280         videoId,
 281         language: subtitle.language,
 282         filename: VideoCaptionModel.generateCaptionName(subtitle.language)
 283       }) as MVideoCaption
 284
 285       // Move physical file
 286       await moveAndProcessCaptionFile(subtitle, videoCaption)
 287
 288       await sequelizeTypescript.transaction(async t => {
 289         await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
 290       })
 291     }
 292   } catch (err) {
 293     logger.warn('Cannot get video subtitles.', { err })
 294   }
 295 }
 296
 297 async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
 298   const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
 299   const uniqHosts = new Set(hosts)
 300
 301   for (const h of uniqHosts) {
 302     if (await isResolvingToUnicastOnly(h) !== true) {
 303       return false
 304     }
 305   }
 306
 307   return true
 308 }