server/controllers/api/videos/import.ts

   1 import express from 'express'
   2 import { move, readFile } from 'fs-extra'
   3 import { decode } from 'magnet-uri'
   4 import parseTorrent, { Instance } from 'parse-torrent'
   5 import { join } from 'path'
   6 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
   7 import { ServerConfigManager } from '@server/lib/server-config-manager'
   8 import { setVideoTags } from '@server/lib/video'
   9 import { FilteredModelAttributes } from '@server/types'
  10 import {
  11   MChannelAccountDefault,
  12   MThumbnail,
  13   MUser,
  14   MVideoAccountDefault,
  15   MVideoCaption,
  16   MVideoTag,
  17   MVideoThumbnail,
  18   MVideoWithBlacklistLight
  19 } from '@server/types/models'
  20 import { MVideoImportFormattable } from '@server/types/models/video/video-import'
  21 import { ServerErrorCode, VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
  22 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
  23 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
  24 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
  25 import { isArray } from '../../../helpers/custom-validators/misc'
  26 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
  27 import { logger } from '../../../helpers/logger'
  28 import { getSecureTorrentName } from '../../../helpers/utils'
  29 import { YoutubeDLWrapper, YoutubeDLInfo } from '../../../helpers/youtube-dl'
  30 import { CONFIG } from '../../../initializers/config'
  31 import { MIMETYPES } from '../../../initializers/constants'
  32 import { sequelizeTypescript } from '../../../initializers/database'
  33 import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
  34 import { JobQueue } from '../../../lib/job-queue/job-queue'
  35 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
  36 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
  37 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
  38 import { VideoModel } from '../../../models/video/video'
  39 import { VideoCaptionModel } from '../../../models/video/video-caption'
  40 import { VideoImportModel } from '../../../models/video/video-import'
  41 import { Hooks } from '@server/lib/plugins/hooks'
  42
  43 const auditLogger = auditLoggerFactory('video-imports')
  44 const videoImportsRouter = express.Router()
  45
  46 const reqVideoFileImport = createReqFiles(
  47   [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
  48   Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
  49   {
  50     thumbnailfile: CONFIG.STORAGE.TMP_DIR,
  51     previewfile: CONFIG.STORAGE.TMP_DIR,
  52     torrentfile: CONFIG.STORAGE.TMP_DIR
  53   }
  54 )
  55
  56 videoImportsRouter.post('/imports',
  57   authenticate,
  58   reqVideoFileImport,
  59   asyncMiddleware(videoImportAddValidator),
  60   asyncRetryTransactionMiddleware(addVideoImport)
  61 )
  62
  63 // ---------------------------------------------------------------------------
  64
  65 export {
  66   videoImportsRouter
  67 }
  68
  69 // ---------------------------------------------------------------------------
  70
  71 function addVideoImport (req: express.Request, res: express.Response) {
  72   if (req.body.targetUrl) return addYoutubeDLImport(req, res)
  73
  74   const file = req.files?.['torrentfile']?.[0]
  75   if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
  76 }
  77
  78 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
  79   const body: VideoImportCreate = req.body
  80   const user = res.locals.oauth.token.User
  81
  82   let videoName: string
  83   let torrentName: string
  84   let magnetUri: string
  85
  86   if (torrentfile) {
  87     const result = await processTorrentOrAbortRequest(req, res, torrentfile)
  88     if (!result) return
  89
  90     videoName = result.name
  91     torrentName = result.torrentName
  92   } else {
  93     const result = processMagnetURI(body)
  94     magnetUri = result.magnetUri
  95     videoName = result.name
  96   }
  97
  98   const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
  99
 100   const thumbnailModel = await processThumbnail(req, video)
 101   const previewModel = await processPreview(req, video)
 102
 103   const videoImport = await insertIntoDB({
 104     video,
 105     thumbnailModel,
 106     previewModel,
 107     videoChannel: res.locals.videoChannel,
 108     tags: body.tags || undefined,
 109     user,
 110     videoImportAttributes: {
 111       magnetUri,
 112       torrentName,
 113       state: VideoImportState.PENDING,
 114       userId: user.id
 115     }
 116   })
 117
 118   // Create job to import the video
 119   const payload = {
 120     type: torrentfile
 121       ? 'torrent-file' as 'torrent-file'
 122       : 'magnet-uri' as 'magnet-uri',
 123     videoImportId: videoImport.id,
 124     magnetUri
 125   }
 126   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 127
 128   auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 129
 130   return res.json(videoImport.toFormattedJSON()).end()
 131 }
 132
 133 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
 134   const body: VideoImportCreate = req.body
 135   const targetUrl = body.targetUrl
 136   const user = res.locals.oauth.token.User
 137
 138   const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
 139
 140   // Get video infos
 141   let youtubeDLInfo: YoutubeDLInfo
 142   try {
 143     youtubeDLInfo = await youtubeDL.getInfoForDownload()
 144   } catch (err) {
 145     logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
 146
 147     return res.fail({
 148       message: 'Cannot fetch remote information of this URL.',
 149       data: {
 150         targetUrl
 151       }
 152     })
 153   }
 154
 155   const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
 156
 157   // Process video thumbnail from request.files
 158   let thumbnailModel = await processThumbnail(req, video)
 159
 160   // Process video thumbnail from url if processing from request.files failed
 161   if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
 162     try {
 163       thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
 164     } catch (err) {
 165       logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
 166     }
 167   }
 168
 169   // Process video preview from request.files
 170   let previewModel = await processPreview(req, video)
 171
 172   // Process video preview from url if processing from request.files failed
 173   if (!previewModel && youtubeDLInfo.thumbnailUrl) {
 174     try {
 175       previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
 176     } catch (err) {
 177       logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
 178     }
 179   }
 180
 181   const videoImport = await insertIntoDB({
 182     video,
 183     thumbnailModel,
 184     previewModel,
 185     videoChannel: res.locals.videoChannel,
 186     tags: body.tags || youtubeDLInfo.tags,
 187     user,
 188     videoImportAttributes: {
 189       targetUrl,
 190       state: VideoImportState.PENDING,
 191       userId: user.id
 192     }
 193   })
 194
 195   // Get video subtitles
 196   await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
 197
 198   let fileExt = `.${youtubeDLInfo.ext}`
 199   if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
 200
 201   // Create job to import the video
 202   const payload = {
 203     type: 'youtube-dl' as 'youtube-dl',
 204     videoImportId: videoImport.id,
 205     fileExt
 206   }
 207   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 208
 209   auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 210
 211   return res.json(videoImport.toFormattedJSON()).end()
 212 }
 213
 214 async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
 215   let videoData = {
 216     name: body.name || importData.name || 'Unknown name',
 217     remote: false,
 218     category: body.category || importData.category,
 219     licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
 220     language: body.language || importData.language,
 221     commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
 222     downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
 223     waitTranscoding: body.waitTranscoding || false,
 224     state: VideoState.TO_IMPORT,
 225     nsfw: body.nsfw || importData.nsfw || false,
 226     description: body.description || importData.description,
 227     support: body.support || null,
 228     privacy: body.privacy || VideoPrivacy.PRIVATE,
 229     duration: 0, // duration will be set by the import job
 230     channelId: channelId,
 231     originallyPublishedAt: body.originallyPublishedAt
 232       ? new Date(body.originallyPublishedAt)
 233       : importData.originallyPublishedAt
 234   }
 235
 236   videoData = await Hooks.wrapObject(
 237     videoData,
 238     body.targetUrl
 239       ? 'filter:api.video.import-url.video-attribute.result'
 240       : 'filter:api.video.import-torrent.video-attribute.result'
 241   )
 242
 243   const video = new VideoModel(videoData)
 244   video.url = getLocalVideoActivityPubUrl(video)
 245
 246   return video
 247 }
 248
 249 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
 250   const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
 251   if (thumbnailField) {
 252     const thumbnailPhysicalFile = thumbnailField[0]
 253
 254     return updateVideoMiniatureFromExisting({
 255       inputPath: thumbnailPhysicalFile.path,
 256       video,
 257       type: ThumbnailType.MINIATURE,
 258       automaticallyGenerated: false
 259     })
 260   }
 261
 262   return undefined
 263 }
 264
 265 async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> {
 266   const previewField = req.files ? req.files['previewfile'] : undefined
 267   if (previewField) {
 268     const previewPhysicalFile = previewField[0]
 269
 270     return updateVideoMiniatureFromExisting({
 271       inputPath: previewPhysicalFile.path,
 272       video,
 273       type: ThumbnailType.PREVIEW,
 274       automaticallyGenerated: false
 275     })
 276   }
 277
 278   return undefined
 279 }
 280
 281 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
 282   try {
 283     return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
 284   } catch (err) {
 285     logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
 286     return undefined
 287   }
 288 }
 289
 290 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
 291   try {
 292     return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
 293   } catch (err) {
 294     logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
 295     return undefined
 296   }
 297 }
 298
 299 async function insertIntoDB (parameters: {
 300   video: MVideoThumbnail
 301   thumbnailModel: MThumbnail
 302   previewModel: MThumbnail
 303   videoChannel: MChannelAccountDefault
 304   tags: string[]
 305   videoImportAttributes: FilteredModelAttributes<VideoImportModel>
 306   user: MUser
 307 }): Promise<MVideoImportFormattable> {
 308   const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
 309
 310   const videoImport = await sequelizeTypescript.transaction(async t => {
 311     const sequelizeOptions = { transaction: t }
 312
 313     // Save video object in database
 314     const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
 315     videoCreated.VideoChannel = videoChannel
 316
 317     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 318     if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 319
 320     await autoBlacklistVideoIfNeeded({
 321       video: videoCreated,
 322       user,
 323       notify: false,
 324       isRemote: false,
 325       isNew: true,
 326       transaction: t
 327     })
 328
 329     await setVideoTags({ video: videoCreated, tags, transaction: t })
 330
 331     // Create video import object in database
 332     const videoImport = await VideoImportModel.create(
 333       Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
 334       sequelizeOptions
 335     ) as MVideoImportFormattable
 336     videoImport.Video = videoCreated
 337
 338     return videoImport
 339   })
 340
 341   return videoImport
 342 }
 343
 344 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
 345   const torrentName = torrentfile.originalname
 346
 347   // Rename the torrent to a secured name
 348   const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
 349   await move(torrentfile.path, newTorrentPath, { overwrite: true })
 350   torrentfile.path = newTorrentPath
 351
 352   const buf = await readFile(torrentfile.path)
 353   const parsedTorrent = parseTorrent(buf) as Instance
 354
 355   if (parsedTorrent.files.length !== 1) {
 356     cleanUpReqFiles(req)
 357
 358     res.fail({
 359       type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT,
 360       message: 'Torrents with only 1 file are supported.'
 361     })
 362     return undefined
 363   }
 364
 365   return {
 366     name: extractNameFromArray(parsedTorrent.name),
 367     torrentName
 368   }
 369 }
 370
 371 function processMagnetURI (body: VideoImportCreate) {
 372   const magnetUri = body.magnetUri
 373   const parsed = decode(magnetUri)
 374
 375   return {
 376     name: extractNameFromArray(parsed.name),
 377     magnetUri
 378   }
 379 }
 380
 381 function extractNameFromArray (name: string | string[]) {
 382   return isArray(name) ? name[0] : name
 383 }
 384
 385 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
 386   try {
 387     const subtitles = await youtubeDL.getSubtitles()
 388
 389     logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
 390
 391     for (const subtitle of subtitles) {
 392       const videoCaption = new VideoCaptionModel({
 393         videoId,
 394         language: subtitle.language,
 395         filename: VideoCaptionModel.generateCaptionName(subtitle.language)
 396       }) as MVideoCaption
 397
 398       // Move physical file
 399       await moveAndProcessCaptionFile(subtitle, videoCaption)
 400
 401       await sequelizeTypescript.transaction(async t => {
 402         await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
 403       })
 404     }
 405   } catch (err) {
 406     logger.warn('Cannot get video subtitles.', { err })
 407   }
 408 }