server/controllers/api/videos/import.ts

   1 import express from 'express'
   2 import { move, readFile, remove } from 'fs-extra'
   3 import { decode } from 'magnet-uri'
   4 import parseTorrent, { Instance } from 'parse-torrent'
   5 import { join } from 'path'
   6 import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
   7 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
   8 import { isResolvingToUnicastOnly } from '@server/helpers/dns'
   9 import { Hooks } from '@server/lib/plugins/hooks'
  10 import { ServerConfigManager } from '@server/lib/server-config-manager'
  11 import { setVideoTags } from '@server/lib/video'
  12 import { FilteredModelAttributes } from '@server/types'
  13 import {
  14   MChannelAccountDefault,
  15   MThumbnail,
  16   MUser,
  17   MVideoAccountDefault,
  18   MVideoCaption,
  19   MVideoTag,
  20   MVideoThumbnail,
  21   MVideoWithBlacklistLight
  22 } from '@server/types/models'
  23 import { MVideoImportFormattable } from '@server/types/models/video/video-import'
  24 import {
  25   HttpStatusCode,
  26   ServerErrorCode,
  27   ThumbnailType,
  28   VideoImportCreate,
  29   VideoImportState,
  30   VideoPrivacy,
  31   VideoState
  32 } from '@shared/models'
  33 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
  34 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
  35 import { isArray } from '../../../helpers/custom-validators/misc'
  36 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
  37 import { logger } from '../../../helpers/logger'
  38 import { getSecureTorrentName } from '../../../helpers/utils'
  39 import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
  40 import { CONFIG } from '../../../initializers/config'
  41 import { MIMETYPES } from '../../../initializers/constants'
  42 import { sequelizeTypescript } from '../../../initializers/database'
  43 import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
  44 import { JobQueue } from '../../../lib/job-queue/job-queue'
  45 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
  46 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
  47 import {
  48   asyncMiddleware,
  49   asyncRetryTransactionMiddleware,
  50   authenticate,
  51   videoImportAddValidator,
  52   videoImportCancelValidator,
  53   videoImportDeleteValidator
  54 } from '../../../middlewares'
  55 import { VideoModel } from '../../../models/video/video'
  56 import { VideoCaptionModel } from '../../../models/video/video-caption'
  57 import { VideoImportModel } from '../../../models/video/video-import'
  58
  59 const auditLogger = auditLoggerFactory('video-imports')
  60 const videoImportsRouter = express.Router()
  61
  62 const reqVideoFileImport = createReqFiles(
  63   [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
  64   { ...MIMETYPES.TORRENT.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT }
  65 )
  66
  67 videoImportsRouter.post('/imports',
  68   authenticate,
  69   reqVideoFileImport,
  70   asyncMiddleware(videoImportAddValidator),
  71   asyncRetryTransactionMiddleware(addVideoImport)
  72 )
  73
  74 videoImportsRouter.post('/imports/:id/cancel',
  75   authenticate,
  76   asyncMiddleware(videoImportCancelValidator),
  77   asyncRetryTransactionMiddleware(cancelVideoImport)
  78 )
  79
  80 videoImportsRouter.delete('/imports/:id',
  81   authenticate,
  82   asyncMiddleware(videoImportDeleteValidator),
  83   asyncRetryTransactionMiddleware(deleteVideoImport)
  84 )
  85
  86 // ---------------------------------------------------------------------------
  87
  88 export {
  89   videoImportsRouter
  90 }
  91
  92 // ---------------------------------------------------------------------------
  93
  94 async function deleteVideoImport (req: express.Request, res: express.Response) {
  95   const videoImport = res.locals.videoImport
  96
  97   await videoImport.destroy()
  98
  99   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 100 }
 101
 102 async function cancelVideoImport (req: express.Request, res: express.Response) {
 103   const videoImport = res.locals.videoImport
 104
 105   videoImport.state = VideoImportState.CANCELLED
 106   await videoImport.save()
 107
 108   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 109 }
 110
 111 function addVideoImport (req: express.Request, res: express.Response) {
 112   if (req.body.targetUrl) return addYoutubeDLImport(req, res)
 113
 114   const file = req.files?.['torrentfile']?.[0]
 115   if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
 116 }
 117
 118 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
 119   const body: VideoImportCreate = req.body
 120   const user = res.locals.oauth.token.User
 121
 122   let videoName: string
 123   let torrentName: string
 124   let magnetUri: string
 125
 126   if (torrentfile) {
 127     const result = await processTorrentOrAbortRequest(req, res, torrentfile)
 128     if (!result) return
 129
 130     videoName = result.name
 131     torrentName = result.torrentName
 132   } else {
 133     const result = processMagnetURI(body)
 134     magnetUri = result.magnetUri
 135     videoName = result.name
 136   }
 137
 138   const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
 139
 140   const thumbnailModel = await processThumbnail(req, video)
 141   const previewModel = await processPreview(req, video)
 142
 143   const videoImport = await insertIntoDB({
 144     video,
 145     thumbnailModel,
 146     previewModel,
 147     videoChannel: res.locals.videoChannel,
 148     tags: body.tags || undefined,
 149     user,
 150     videoImportAttributes: {
 151       magnetUri,
 152       torrentName,
 153       state: VideoImportState.PENDING,
 154       userId: user.id
 155     }
 156   })
 157
 158   // Create job to import the video
 159   const payload = {
 160     type: torrentfile
 161       ? 'torrent-file' as 'torrent-file'
 162       : 'magnet-uri' as 'magnet-uri',
 163     videoImportId: videoImport.id,
 164     magnetUri
 165   }
 166   await JobQueue.Instance.createJob({ type: 'video-import', payload })
 167
 168   auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 169
 170   return res.json(videoImport.toFormattedJSON()).end()
 171 }
 172
 173 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
 174   const body: VideoImportCreate = req.body
 175   const targetUrl = body.targetUrl
 176   const user = res.locals.oauth.token.User
 177
 178   const youtubeDL = new YoutubeDLWrapper(
 179     targetUrl,
 180     ServerConfigManager.Instance.getEnabledResolutions('vod'),
 181     CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
 182   )
 183
 184   // Get video infos
 185   let youtubeDLInfo: YoutubeDLInfo
 186   try {
 187     youtubeDLInfo = await youtubeDL.getInfoForDownload()
 188   } catch (err) {
 189     logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
 190
 191     return res.fail({
 192       message: 'Cannot fetch remote information of this URL.',
 193       data: {
 194         targetUrl
 195       }
 196     })
 197   }
 198
 199   if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
 200     return res.fail({
 201       status: HttpStatusCode.FORBIDDEN_403,
 202       message: 'Cannot use non unicast IP as targetUrl.'
 203     })
 204   }
 205
 206   const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
 207
 208   // Process video thumbnail from request.files
 209   let thumbnailModel = await processThumbnail(req, video)
 210
 211   // Process video thumbnail from url if processing from request.files failed
 212   if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
 213     try {
 214       thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
 215     } catch (err) {
 216       logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
 217     }
 218   }
 219
 220   // Process video preview from request.files
 221   let previewModel = await processPreview(req, video)
 222
 223   // Process video preview from url if processing from request.files failed
 224   if (!previewModel && youtubeDLInfo.thumbnailUrl) {
 225     try {
 226       previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
 227     } catch (err) {
 228       logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
 229     }
 230   }
 231
 232   const videoImport = await insertIntoDB({
 233     video,
 234     thumbnailModel,
 235     previewModel,
 236     videoChannel: res.locals.videoChannel,
 237     tags: body.tags || youtubeDLInfo.tags,
 238     user,
 239     videoImportAttributes: {
 240       targetUrl,
 241       state: VideoImportState.PENDING,
 242       userId: user.id
 243     }
 244   })
 245
 246   // Get video subtitles
 247   await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
 248
 249   let fileExt = `.${youtubeDLInfo.ext}`
 250   if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
 251
 252   // Create job to import the video
 253   const payload = {
 254     type: 'youtube-dl' as 'youtube-dl',
 255     videoImportId: videoImport.id,
 256     fileExt
 257   }
 258   await JobQueue.Instance.createJob({ type: 'video-import', payload })
 259
 260   auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 261
 262   return res.json(videoImport.toFormattedJSON()).end()
 263 }
 264
 265 async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
 266   let videoData = {
 267     name: body.name || importData.name || 'Unknown name',
 268     remote: false,
 269     category: body.category || importData.category,
 270     licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
 271     language: body.language || importData.language,
 272     commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
 273     downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
 274     waitTranscoding: body.waitTranscoding || false,
 275     state: VideoState.TO_IMPORT,
 276     nsfw: body.nsfw || importData.nsfw || false,
 277     description: body.description || importData.description,
 278     support: body.support || null,
 279     privacy: body.privacy || VideoPrivacy.PRIVATE,
 280     duration: 0, // duration will be set by the import job
 281     channelId,
 282     originallyPublishedAt: body.originallyPublishedAt
 283       ? new Date(body.originallyPublishedAt)
 284       : importData.originallyPublishedAt
 285   }
 286
 287   videoData = await Hooks.wrapObject(
 288     videoData,
 289     body.targetUrl
 290       ? 'filter:api.video.import-url.video-attribute.result'
 291       : 'filter:api.video.import-torrent.video-attribute.result'
 292   )
 293
 294   const video = new VideoModel(videoData)
 295   video.url = getLocalVideoActivityPubUrl(video)
 296
 297   return video
 298 }
 299
 300 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
 301   const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
 302   if (thumbnailField) {
 303     const thumbnailPhysicalFile = thumbnailField[0]
 304
 305     return updateVideoMiniatureFromExisting({
 306       inputPath: thumbnailPhysicalFile.path,
 307       video,
 308       type: ThumbnailType.MINIATURE,
 309       automaticallyGenerated: false
 310     })
 311   }
 312
 313   return undefined
 314 }
 315
 316 async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> {
 317   const previewField = req.files ? req.files['previewfile'] : undefined
 318   if (previewField) {
 319     const previewPhysicalFile = previewField[0]
 320
 321     return updateVideoMiniatureFromExisting({
 322       inputPath: previewPhysicalFile.path,
 323       video,
 324       type: ThumbnailType.PREVIEW,
 325       automaticallyGenerated: false
 326     })
 327   }
 328
 329   return undefined
 330 }
 331
 332 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
 333   try {
 334     return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
 335   } catch (err) {
 336     logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
 337     return undefined
 338   }
 339 }
 340
 341 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
 342   try {
 343     return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
 344   } catch (err) {
 345     logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
 346     return undefined
 347   }
 348 }
 349
 350 async function insertIntoDB (parameters: {
 351   video: MVideoThumbnail
 352   thumbnailModel: MThumbnail
 353   previewModel: MThumbnail
 354   videoChannel: MChannelAccountDefault
 355   tags: string[]
 356   videoImportAttributes: FilteredModelAttributes<VideoImportModel>
 357   user: MUser
 358 }): Promise<MVideoImportFormattable> {
 359   const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
 360
 361   const videoImport = await sequelizeTypescript.transaction(async t => {
 362     const sequelizeOptions = { transaction: t }
 363
 364     // Save video object in database
 365     const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
 366     videoCreated.VideoChannel = videoChannel
 367
 368     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 369     if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 370
 371     await autoBlacklistVideoIfNeeded({
 372       video: videoCreated,
 373       user,
 374       notify: false,
 375       isRemote: false,
 376       isNew: true,
 377       transaction: t
 378     })
 379
 380     await setVideoTags({ video: videoCreated, tags, transaction: t })
 381
 382     // Create video import object in database
 383     const videoImport = await VideoImportModel.create(
 384       Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
 385       sequelizeOptions
 386     ) as MVideoImportFormattable
 387     videoImport.Video = videoCreated
 388
 389     return videoImport
 390   })
 391
 392   return videoImport
 393 }
 394
 395 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
 396   const torrentName = torrentfile.originalname
 397
 398   // Rename the torrent to a secured name
 399   const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
 400   await move(torrentfile.path, newTorrentPath, { overwrite: true })
 401   torrentfile.path = newTorrentPath
 402
 403   const buf = await readFile(torrentfile.path)
 404   const parsedTorrent = parseTorrent(buf) as Instance
 405
 406   if (parsedTorrent.files.length !== 1) {
 407     cleanUpReqFiles(req)
 408
 409     res.fail({
 410       type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT,
 411       message: 'Torrents with only 1 file are supported.'
 412     })
 413     return undefined
 414   }
 415
 416   return {
 417     name: extractNameFromArray(parsedTorrent.name),
 418     torrentName
 419   }
 420 }
 421
 422 function processMagnetURI (body: VideoImportCreate) {
 423   const magnetUri = body.magnetUri
 424   const parsed = decode(magnetUri)
 425
 426   return {
 427     name: extractNameFromArray(parsed.name),
 428     magnetUri
 429   }
 430 }
 431
 432 function extractNameFromArray (name: string | string[]) {
 433   return isArray(name) ? name[0] : name
 434 }
 435
 436 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
 437   try {
 438     const subtitles = await youtubeDL.getSubtitles()
 439
 440     logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
 441
 442     for (const subtitle of subtitles) {
 443       if (!await isVTTFileValid(subtitle.path)) {
 444         await remove(subtitle.path)
 445         continue
 446       }
 447
 448       const videoCaption = new VideoCaptionModel({
 449         videoId,
 450         language: subtitle.language,
 451         filename: VideoCaptionModel.generateCaptionName(subtitle.language)
 452       }) as MVideoCaption
 453
 454       // Move physical file
 455       await moveAndProcessCaptionFile(subtitle, videoCaption)
 456
 457       await sequelizeTypescript.transaction(async t => {
 458         await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
 459       })
 460     }
 461   } catch (err) {
 462     logger.warn('Cannot get video subtitles.', { err })
 463   }
 464 }
 465
 466 async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
 467   const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
 468   const uniqHosts = new Set(hosts)
 469
 470   for (const h of uniqHosts) {
 471     if (await isResolvingToUnicastOnly(h) !== true) {
 472       return false
 473     }
 474   }
 475
 476   return true
 477 }