server/controllers/api/videos/import.ts

   1 import express from 'express'
   2 import { move, readFile, remove } from 'fs-extra'
   3 import { decode } from 'magnet-uri'
   4 import parseTorrent, { Instance } from 'parse-torrent'
   5 import { join } from 'path'
   6 import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
   7 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
   8 import { isResolvingToUnicastOnly } from '@server/helpers/dns'
   9 import { Hooks } from '@server/lib/plugins/hooks'
  10 import { ServerConfigManager } from '@server/lib/server-config-manager'
  11 import { setVideoTags } from '@server/lib/video'
  12 import { FilteredModelAttributes } from '@server/types'
  13 import {
  14   MChannelAccountDefault,
  15   MThumbnail,
  16   MUser,
  17   MVideoAccountDefault,
  18   MVideoCaption,
  19   MVideoTag,
  20   MVideoThumbnail,
  21   MVideoWithBlacklistLight
  22 } from '@server/types/models'
  23 import { MVideoImportFormattable } from '@server/types/models/video/video-import'
  24 import {
  25   HttpStatusCode,
  26   ServerErrorCode,
  27   ThumbnailType,
  28   VideoImportCreate,
  29   VideoImportState,
  30   VideoPrivacy,
  31   VideoState
  32 } from '@shared/models'
  33 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
  34 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
  35 import { isArray } from '../../../helpers/custom-validators/misc'
  36 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
  37 import { logger } from '../../../helpers/logger'
  38 import { getSecureTorrentName } from '../../../helpers/utils'
  39 import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
  40 import { CONFIG } from '../../../initializers/config'
  41 import { MIMETYPES } from '../../../initializers/constants'
  42 import { sequelizeTypescript } from '../../../initializers/database'
  43 import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
  44 import { JobQueue } from '../../../lib/job-queue/job-queue'
  45 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
  46 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
  47 import {
  48   asyncMiddleware,
  49   asyncRetryTransactionMiddleware,
  50   authenticate,
  51   videoImportAddValidator,
  52   videoImportCancelValidator,
  53   videoImportDeleteValidator
  54 } from '../../../middlewares'
  55 import { VideoModel } from '../../../models/video/video'
  56 import { VideoCaptionModel } from '../../../models/video/video-caption'
  57 import { VideoImportModel } from '../../../models/video/video-import'
  58
  59 const auditLogger = auditLoggerFactory('video-imports')
  60 const videoImportsRouter = express.Router()
  61
  62 const reqVideoFileImport = createReqFiles(
  63   [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
  64   { ...MIMETYPES.TORRENT.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT }
  65 )
  66
  67 videoImportsRouter.post('/imports',
  68   authenticate,
  69   reqVideoFileImport,
  70   asyncMiddleware(videoImportAddValidator),
  71   asyncRetryTransactionMiddleware(addVideoImport)
  72 )
  73
  74 videoImportsRouter.post('/imports/:id/cancel',
  75   authenticate,
  76   asyncMiddleware(videoImportCancelValidator),
  77   asyncRetryTransactionMiddleware(cancelVideoImport)
  78 )
  79
  80 videoImportsRouter.delete('/imports/:id',
  81   authenticate,
  82   asyncMiddleware(videoImportDeleteValidator),
  83   asyncRetryTransactionMiddleware(deleteVideoImport)
  84 )
  85
  86 // ---------------------------------------------------------------------------
  87
  88 export {
  89   videoImportsRouter
  90 }
  91
  92 // ---------------------------------------------------------------------------
  93
  94 async function deleteVideoImport (req: express.Request, res: express.Response) {
  95   const videoImport = res.locals.videoImport
  96
  97   await videoImport.destroy()
  98
  99   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 100 }
 101
 102 async function cancelVideoImport (req: express.Request, res: express.Response) {
 103   const videoImport = res.locals.videoImport
 104
 105   videoImport.state = VideoImportState.CANCELLED
 106   await videoImport.save()
 107
 108   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 109 }
 110
 111 function addVideoImport (req: express.Request, res: express.Response) {
 112   if (req.body.targetUrl) return addYoutubeDLImport(req, res)
 113
 114   const file = req.files?.['torrentfile']?.[0]
 115   if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
 116 }
 117
 118 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
 119   const body: VideoImportCreate = req.body
 120   const user = res.locals.oauth.token.User
 121
 122   let videoName: string
 123   let torrentName: string
 124   let magnetUri: string
 125
 126   if (torrentfile) {
 127     const result = await processTorrentOrAbortRequest(req, res, torrentfile)
 128     if (!result) return
 129
 130     videoName = result.name
 131     torrentName = result.torrentName
 132   } else {
 133     const result = processMagnetURI(body)
 134     magnetUri = result.magnetUri
 135     videoName = result.name
 136   }
 137
 138   const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
 139
 140   const thumbnailModel = await processThumbnail(req, video)
 141   const previewModel = await processPreview(req, video)
 142
 143   const videoImport = await insertIntoDB({
 144     video,
 145     thumbnailModel,
 146     previewModel,
 147     videoChannel: res.locals.videoChannel,
 148     tags: body.tags || undefined,
 149     user,
 150     videoImportAttributes: {
 151       magnetUri,
 152       torrentName,
 153       state: VideoImportState.PENDING,
 154       userId: user.id
 155     }
 156   })
 157
 158   // Create job to import the video
 159   const payload = {
 160     type: torrentfile
 161       ? 'torrent-file' as 'torrent-file'
 162       : 'magnet-uri' as 'magnet-uri',
 163     videoImportId: videoImport.id,
 164     magnetUri
 165   }
 166   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 167
 168   auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 169
 170   return res.json(videoImport.toFormattedJSON()).end()
 171 }
 172
 173 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
 174   const body: VideoImportCreate = req.body
 175   const targetUrl = body.targetUrl
 176   const user = res.locals.oauth.token.User
 177
 178   const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
 179
 180   // Get video infos
 181   let youtubeDLInfo: YoutubeDLInfo
 182   try {
 183     youtubeDLInfo = await youtubeDL.getInfoForDownload()
 184   } catch (err) {
 185     logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
 186
 187     return res.fail({
 188       message: 'Cannot fetch remote information of this URL.',
 189       data: {
 190         targetUrl
 191       }
 192     })
 193   }
 194
 195   if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
 196     return res.fail({
 197       status: HttpStatusCode.FORBIDDEN_403,
 198       message: 'Cannot use non unicast IP as targetUrl.'
 199     })
 200   }
 201
 202   const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
 203
 204   // Process video thumbnail from request.files
 205   let thumbnailModel = await processThumbnail(req, video)
 206
 207   // Process video thumbnail from url if processing from request.files failed
 208   if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
 209     try {
 210       thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
 211     } catch (err) {
 212       logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
 213     }
 214   }
 215
 216   // Process video preview from request.files
 217   let previewModel = await processPreview(req, video)
 218
 219   // Process video preview from url if processing from request.files failed
 220   if (!previewModel && youtubeDLInfo.thumbnailUrl) {
 221     try {
 222       previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
 223     } catch (err) {
 224       logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
 225     }
 226   }
 227
 228   const videoImport = await insertIntoDB({
 229     video,
 230     thumbnailModel,
 231     previewModel,
 232     videoChannel: res.locals.videoChannel,
 233     tags: body.tags || youtubeDLInfo.tags,
 234     user,
 235     videoImportAttributes: {
 236       targetUrl,
 237       state: VideoImportState.PENDING,
 238       userId: user.id
 239     }
 240   })
 241
 242   // Get video subtitles
 243   await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
 244
 245   let fileExt = `.${youtubeDLInfo.ext}`
 246   if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
 247
 248   // Create job to import the video
 249   const payload = {
 250     type: 'youtube-dl' as 'youtube-dl',
 251     videoImportId: videoImport.id,
 252     fileExt
 253   }
 254   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 255
 256   auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 257
 258   return res.json(videoImport.toFormattedJSON()).end()
 259 }
 260
 261 async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
 262   let videoData = {
 263     name: body.name || importData.name || 'Unknown name',
 264     remote: false,
 265     category: body.category || importData.category,
 266     licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
 267     language: body.language || importData.language,
 268     commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
 269     downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
 270     waitTranscoding: body.waitTranscoding || false,
 271     state: VideoState.TO_IMPORT,
 272     nsfw: body.nsfw || importData.nsfw || false,
 273     description: body.description || importData.description,
 274     support: body.support || null,
 275     privacy: body.privacy || VideoPrivacy.PRIVATE,
 276     duration: 0, // duration will be set by the import job
 277     channelId,
 278     originallyPublishedAt: body.originallyPublishedAt
 279       ? new Date(body.originallyPublishedAt)
 280       : importData.originallyPublishedAt
 281   }
 282
 283   videoData = await Hooks.wrapObject(
 284     videoData,
 285     body.targetUrl
 286       ? 'filter:api.video.import-url.video-attribute.result'
 287       : 'filter:api.video.import-torrent.video-attribute.result'
 288   )
 289
 290   const video = new VideoModel(videoData)
 291   video.url = getLocalVideoActivityPubUrl(video)
 292
 293   return video
 294 }
 295
 296 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
 297   const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
 298   if (thumbnailField) {
 299     const thumbnailPhysicalFile = thumbnailField[0]
 300
 301     return updateVideoMiniatureFromExisting({
 302       inputPath: thumbnailPhysicalFile.path,
 303       video,
 304       type: ThumbnailType.MINIATURE,
 305       automaticallyGenerated: false
 306     })
 307   }
 308
 309   return undefined
 310 }
 311
 312 async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> {
 313   const previewField = req.files ? req.files['previewfile'] : undefined
 314   if (previewField) {
 315     const previewPhysicalFile = previewField[0]
 316
 317     return updateVideoMiniatureFromExisting({
 318       inputPath: previewPhysicalFile.path,
 319       video,
 320       type: ThumbnailType.PREVIEW,
 321       automaticallyGenerated: false
 322     })
 323   }
 324
 325   return undefined
 326 }
 327
 328 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
 329   try {
 330     return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
 331   } catch (err) {
 332     logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
 333     return undefined
 334   }
 335 }
 336
 337 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
 338   try {
 339     return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
 340   } catch (err) {
 341     logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
 342     return undefined
 343   }
 344 }
 345
 346 async function insertIntoDB (parameters: {
 347   video: MVideoThumbnail
 348   thumbnailModel: MThumbnail
 349   previewModel: MThumbnail
 350   videoChannel: MChannelAccountDefault
 351   tags: string[]
 352   videoImportAttributes: FilteredModelAttributes<VideoImportModel>
 353   user: MUser
 354 }): Promise<MVideoImportFormattable> {
 355   const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
 356
 357   const videoImport = await sequelizeTypescript.transaction(async t => {
 358     const sequelizeOptions = { transaction: t }
 359
 360     // Save video object in database
 361     const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
 362     videoCreated.VideoChannel = videoChannel
 363
 364     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 365     if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 366
 367     await autoBlacklistVideoIfNeeded({
 368       video: videoCreated,
 369       user,
 370       notify: false,
 371       isRemote: false,
 372       isNew: true,
 373       transaction: t
 374     })
 375
 376     await setVideoTags({ video: videoCreated, tags, transaction: t })
 377
 378     // Create video import object in database
 379     const videoImport = await VideoImportModel.create(
 380       Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
 381       sequelizeOptions
 382     ) as MVideoImportFormattable
 383     videoImport.Video = videoCreated
 384
 385     return videoImport
 386   })
 387
 388   return videoImport
 389 }
 390
 391 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
 392   const torrentName = torrentfile.originalname
 393
 394   // Rename the torrent to a secured name
 395   const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
 396   await move(torrentfile.path, newTorrentPath, { overwrite: true })
 397   torrentfile.path = newTorrentPath
 398
 399   const buf = await readFile(torrentfile.path)
 400   const parsedTorrent = parseTorrent(buf) as Instance
 401
 402   if (parsedTorrent.files.length !== 1) {
 403     cleanUpReqFiles(req)
 404
 405     res.fail({
 406       type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT,
 407       message: 'Torrents with only 1 file are supported.'
 408     })
 409     return undefined
 410   }
 411
 412   return {
 413     name: extractNameFromArray(parsedTorrent.name),
 414     torrentName
 415   }
 416 }
 417
 418 function processMagnetURI (body: VideoImportCreate) {
 419   const magnetUri = body.magnetUri
 420   const parsed = decode(magnetUri)
 421
 422   return {
 423     name: extractNameFromArray(parsed.name),
 424     magnetUri
 425   }
 426 }
 427
 428 function extractNameFromArray (name: string | string[]) {
 429   return isArray(name) ? name[0] : name
 430 }
 431
 432 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
 433   try {
 434     const subtitles = await youtubeDL.getSubtitles()
 435
 436     logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
 437
 438     for (const subtitle of subtitles) {
 439       if (!await isVTTFileValid(subtitle.path)) {
 440         await remove(subtitle.path)
 441         continue
 442       }
 443
 444       const videoCaption = new VideoCaptionModel({
 445         videoId,
 446         language: subtitle.language,
 447         filename: VideoCaptionModel.generateCaptionName(subtitle.language)
 448       }) as MVideoCaption
 449
 450       // Move physical file
 451       await moveAndProcessCaptionFile(subtitle, videoCaption)
 452
 453       await sequelizeTypescript.transaction(async t => {
 454         await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
 455       })
 456     }
 457   } catch (err) {
 458     logger.warn('Cannot get video subtitles.', { err })
 459   }
 460 }
 461
 462 async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
 463   const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
 464   const uniqHosts = new Set(hosts)
 465
 466   for (const h of uniqHosts) {
 467     if (await isResolvingToUnicastOnly(h) !== true) {
 468       return false
 469     }
 470   }
 471
 472   return true
 473 }