server/controllers/api/videos/import.ts

   1 import express from 'express'
   2 import { move, readFile, remove } from 'fs-extra'
   3 import { decode } from 'magnet-uri'
   4 import parseTorrent, { Instance } from 'parse-torrent'
   5 import { join } from 'path'
   6 import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
   7 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
   8 import { isResolvingToUnicastOnly } from '@server/helpers/dns'
   9 import { Hooks } from '@server/lib/plugins/hooks'
  10 import { ServerConfigManager } from '@server/lib/server-config-manager'
  11 import { setVideoTags } from '@server/lib/video'
  12 import { FilteredModelAttributes } from '@server/types'
  13 import {
  14   MChannelAccountDefault,
  15   MThumbnail,
  16   MUser,
  17   MVideoAccountDefault,
  18   MVideoCaption,
  19   MVideoTag,
  20   MVideoThumbnail,
  21   MVideoWithBlacklistLight
  22 } from '@server/types/models'
  23 import { MVideoImportFormattable } from '@server/types/models/video/video-import'
  24 import {
  25   HttpStatusCode,
  26   ServerErrorCode,
  27   ThumbnailType,
  28   VideoImportCreate,
  29   VideoImportState,
  30   VideoPrivacy,
  31   VideoState
  32 } from '@shared/models'
  33 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
  34 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
  35 import { isArray } from '../../../helpers/custom-validators/misc'
  36 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
  37 import { logger } from '../../../helpers/logger'
  38 import { getSecureTorrentName } from '../../../helpers/utils'
  39 import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
  40 import { CONFIG } from '../../../initializers/config'
  41 import { MIMETYPES } from '../../../initializers/constants'
  42 import { sequelizeTypescript } from '../../../initializers/database'
  43 import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
  44 import { JobQueue } from '../../../lib/job-queue/job-queue'
  45 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
  46 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
  47 import {
  48   asyncMiddleware,
  49   asyncRetryTransactionMiddleware,
  50   authenticate,
  51   videoImportAddValidator,
  52   videoImportCancelValidator,
  53   videoImportDeleteValidator
  54 } from '../../../middlewares'
  55 import { VideoModel } from '../../../models/video/video'
  56 import { VideoCaptionModel } from '../../../models/video/video-caption'
  57 import { VideoImportModel } from '../../../models/video/video-import'
  58
  59 const auditLogger = auditLoggerFactory('video-imports')
  60 const videoImportsRouter = express.Router()
  61
  62 const reqVideoFileImport = createReqFiles(
  63   [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
  64   Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
  65   {
  66     thumbnailfile: CONFIG.STORAGE.TMP_DIR,
  67     previewfile: CONFIG.STORAGE.TMP_DIR,
  68     torrentfile: CONFIG.STORAGE.TMP_DIR
  69   }
  70 )
  71
  72 videoImportsRouter.post('/imports',
  73   authenticate,
  74   reqVideoFileImport,
  75   asyncMiddleware(videoImportAddValidator),
  76   asyncRetryTransactionMiddleware(addVideoImport)
  77 )
  78
  79 videoImportsRouter.post('/imports/:id/cancel',
  80   authenticate,
  81   asyncMiddleware(videoImportCancelValidator),
  82   asyncRetryTransactionMiddleware(cancelVideoImport)
  83 )
  84
  85 videoImportsRouter.delete('/imports/:id',
  86   authenticate,
  87   asyncMiddleware(videoImportDeleteValidator),
  88   asyncRetryTransactionMiddleware(deleteVideoImport)
  89 )
  90
  91 // ---------------------------------------------------------------------------
  92
  93 export {
  94   videoImportsRouter
  95 }
  96
  97 // ---------------------------------------------------------------------------
  98
  99 async function deleteVideoImport (req: express.Request, res: express.Response) {
 100   const videoImport = res.locals.videoImport
 101
 102   await videoImport.destroy()
 103
 104   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 105 }
 106
 107 async function cancelVideoImport (req: express.Request, res: express.Response) {
 108   const videoImport = res.locals.videoImport
 109
 110   videoImport.state = VideoImportState.CANCELLED
 111   await videoImport.save()
 112
 113   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 114 }
 115
 116 function addVideoImport (req: express.Request, res: express.Response) {
 117   if (req.body.targetUrl) return addYoutubeDLImport(req, res)
 118
 119   const file = req.files?.['torrentfile']?.[0]
 120   if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
 121 }
 122
 123 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
 124   const body: VideoImportCreate = req.body
 125   const user = res.locals.oauth.token.User
 126
 127   let videoName: string
 128   let torrentName: string
 129   let magnetUri: string
 130
 131   if (torrentfile) {
 132     const result = await processTorrentOrAbortRequest(req, res, torrentfile)
 133     if (!result) return
 134
 135     videoName = result.name
 136     torrentName = result.torrentName
 137   } else {
 138     const result = processMagnetURI(body)
 139     magnetUri = result.magnetUri
 140     videoName = result.name
 141   }
 142
 143   const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
 144
 145   const thumbnailModel = await processThumbnail(req, video)
 146   const previewModel = await processPreview(req, video)
 147
 148   const videoImport = await insertIntoDB({
 149     video,
 150     thumbnailModel,
 151     previewModel,
 152     videoChannel: res.locals.videoChannel,
 153     tags: body.tags || undefined,
 154     user,
 155     videoImportAttributes: {
 156       magnetUri,
 157       torrentName,
 158       state: VideoImportState.PENDING,
 159       userId: user.id
 160     }
 161   })
 162
 163   // Create job to import the video
 164   const payload = {
 165     type: torrentfile
 166       ? 'torrent-file' as 'torrent-file'
 167       : 'magnet-uri' as 'magnet-uri',
 168     videoImportId: videoImport.id,
 169     magnetUri
 170   }
 171   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 172
 173   auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 174
 175   return res.json(videoImport.toFormattedJSON()).end()
 176 }
 177
 178 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
 179   const body: VideoImportCreate = req.body
 180   const targetUrl = body.targetUrl
 181   const user = res.locals.oauth.token.User
 182
 183   const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
 184
 185   // Get video infos
 186   let youtubeDLInfo: YoutubeDLInfo
 187   try {
 188     youtubeDLInfo = await youtubeDL.getInfoForDownload()
 189   } catch (err) {
 190     logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
 191
 192     return res.fail({
 193       message: 'Cannot fetch remote information of this URL.',
 194       data: {
 195         targetUrl
 196       }
 197     })
 198   }
 199
 200   if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
 201     return res.fail({
 202       status: HttpStatusCode.FORBIDDEN_403,
 203       message: 'Cannot use non unicast IP as targetUrl.'
 204     })
 205   }
 206
 207   const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
 208
 209   // Process video thumbnail from request.files
 210   let thumbnailModel = await processThumbnail(req, video)
 211
 212   // Process video thumbnail from url if processing from request.files failed
 213   if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
 214     try {
 215       thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
 216     } catch (err) {
 217       logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
 218     }
 219   }
 220
 221   // Process video preview from request.files
 222   let previewModel = await processPreview(req, video)
 223
 224   // Process video preview from url if processing from request.files failed
 225   if (!previewModel && youtubeDLInfo.thumbnailUrl) {
 226     try {
 227       previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
 228     } catch (err) {
 229       logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
 230     }
 231   }
 232
 233   const videoImport = await insertIntoDB({
 234     video,
 235     thumbnailModel,
 236     previewModel,
 237     videoChannel: res.locals.videoChannel,
 238     tags: body.tags || youtubeDLInfo.tags,
 239     user,
 240     videoImportAttributes: {
 241       targetUrl,
 242       state: VideoImportState.PENDING,
 243       userId: user.id
 244     }
 245   })
 246
 247   // Get video subtitles
 248   await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
 249
 250   let fileExt = `.${youtubeDLInfo.ext}`
 251   if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
 252
 253   // Create job to import the video
 254   const payload = {
 255     type: 'youtube-dl' as 'youtube-dl',
 256     videoImportId: videoImport.id,
 257     fileExt
 258   }
 259   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 260
 261   auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 262
 263   return res.json(videoImport.toFormattedJSON()).end()
 264 }
 265
 266 async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
 267   let videoData = {
 268     name: body.name || importData.name || 'Unknown name',
 269     remote: false,
 270     category: body.category || importData.category,
 271     licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
 272     language: body.language || importData.language,
 273     commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
 274     downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
 275     waitTranscoding: body.waitTranscoding || false,
 276     state: VideoState.TO_IMPORT,
 277     nsfw: body.nsfw || importData.nsfw || false,
 278     description: body.description || importData.description,
 279     support: body.support || null,
 280     privacy: body.privacy || VideoPrivacy.PRIVATE,
 281     duration: 0, // duration will be set by the import job
 282     channelId: channelId,
 283     originallyPublishedAt: body.originallyPublishedAt
 284       ? new Date(body.originallyPublishedAt)
 285       : importData.originallyPublishedAt
 286   }
 287
 288   videoData = await Hooks.wrapObject(
 289     videoData,
 290     body.targetUrl
 291       ? 'filter:api.video.import-url.video-attribute.result'
 292       : 'filter:api.video.import-torrent.video-attribute.result'
 293   )
 294
 295   const video = new VideoModel(videoData)
 296   video.url = getLocalVideoActivityPubUrl(video)
 297
 298   return video
 299 }
 300
 301 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
 302   const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
 303   if (thumbnailField) {
 304     const thumbnailPhysicalFile = thumbnailField[0]
 305
 306     return updateVideoMiniatureFromExisting({
 307       inputPath: thumbnailPhysicalFile.path,
 308       video,
 309       type: ThumbnailType.MINIATURE,
 310       automaticallyGenerated: false
 311     })
 312   }
 313
 314   return undefined
 315 }
 316
 317 async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> {
 318   const previewField = req.files ? req.files['previewfile'] : undefined
 319   if (previewField) {
 320     const previewPhysicalFile = previewField[0]
 321
 322     return updateVideoMiniatureFromExisting({
 323       inputPath: previewPhysicalFile.path,
 324       video,
 325       type: ThumbnailType.PREVIEW,
 326       automaticallyGenerated: false
 327     })
 328   }
 329
 330   return undefined
 331 }
 332
 333 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
 334   try {
 335     return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
 336   } catch (err) {
 337     logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
 338     return undefined
 339   }
 340 }
 341
 342 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
 343   try {
 344     return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
 345   } catch (err) {
 346     logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
 347     return undefined
 348   }
 349 }
 350
 351 async function insertIntoDB (parameters: {
 352   video: MVideoThumbnail
 353   thumbnailModel: MThumbnail
 354   previewModel: MThumbnail
 355   videoChannel: MChannelAccountDefault
 356   tags: string[]
 357   videoImportAttributes: FilteredModelAttributes<VideoImportModel>
 358   user: MUser
 359 }): Promise<MVideoImportFormattable> {
 360   const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
 361
 362   const videoImport = await sequelizeTypescript.transaction(async t => {
 363     const sequelizeOptions = { transaction: t }
 364
 365     // Save video object in database
 366     const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
 367     videoCreated.VideoChannel = videoChannel
 368
 369     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 370     if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 371
 372     await autoBlacklistVideoIfNeeded({
 373       video: videoCreated,
 374       user,
 375       notify: false,
 376       isRemote: false,
 377       isNew: true,
 378       transaction: t
 379     })
 380
 381     await setVideoTags({ video: videoCreated, tags, transaction: t })
 382
 383     // Create video import object in database
 384     const videoImport = await VideoImportModel.create(
 385       Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
 386       sequelizeOptions
 387     ) as MVideoImportFormattable
 388     videoImport.Video = videoCreated
 389
 390     return videoImport
 391   })
 392
 393   return videoImport
 394 }
 395
 396 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
 397   const torrentName = torrentfile.originalname
 398
 399   // Rename the torrent to a secured name
 400   const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
 401   await move(torrentfile.path, newTorrentPath, { overwrite: true })
 402   torrentfile.path = newTorrentPath
 403
 404   const buf = await readFile(torrentfile.path)
 405   const parsedTorrent = parseTorrent(buf) as Instance
 406
 407   if (parsedTorrent.files.length !== 1) {
 408     cleanUpReqFiles(req)
 409
 410     res.fail({
 411       type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT,
 412       message: 'Torrents with only 1 file are supported.'
 413     })
 414     return undefined
 415   }
 416
 417   return {
 418     name: extractNameFromArray(parsedTorrent.name),
 419     torrentName
 420   }
 421 }
 422
 423 function processMagnetURI (body: VideoImportCreate) {
 424   const magnetUri = body.magnetUri
 425   const parsed = decode(magnetUri)
 426
 427   return {
 428     name: extractNameFromArray(parsed.name),
 429     magnetUri
 430   }
 431 }
 432
 433 function extractNameFromArray (name: string | string[]) {
 434   return isArray(name) ? name[0] : name
 435 }
 436
 437 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
 438   try {
 439     const subtitles = await youtubeDL.getSubtitles()
 440
 441     logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
 442
 443     for (const subtitle of subtitles) {
 444       if (!await isVTTFileValid(subtitle.path)) {
 445         await remove(subtitle.path)
 446         continue
 447       }
 448
 449       const videoCaption = new VideoCaptionModel({
 450         videoId,
 451         language: subtitle.language,
 452         filename: VideoCaptionModel.generateCaptionName(subtitle.language)
 453       }) as MVideoCaption
 454
 455       // Move physical file
 456       await moveAndProcessCaptionFile(subtitle, videoCaption)
 457
 458       await sequelizeTypescript.transaction(async t => {
 459         await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
 460       })
 461     }
 462   } catch (err) {
 463     logger.warn('Cannot get video subtitles.', { err })
 464   }
 465 }
 466
 467 async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
 468   const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
 469   const uniqHosts = new Set(hosts)
 470
 471   for (const h of uniqHosts) {
 472     if (await isResolvingToUnicastOnly(h) !== true) {
 473       return false
 474     }
 475   }
 476
 477   return true
 478 }