1 import express from 'express'
2 import { move, readFile, remove } from 'fs-extra'
3 import { decode } from 'magnet-uri'
4 import parseTorrent, { Instance } from 'parse-torrent'
5 import { join } from 'path'
6 import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
7 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
8 import { isResolvingToUnicastOnly } from '@server/helpers/dns'
9 import { Hooks } from '@server/lib/plugins/hooks'
10 import { ServerConfigManager } from '@server/lib/server-config-manager'
11 import { setVideoTags } from '@server/lib/video'
12 import { FilteredModelAttributes } from '@server/types'
14 MChannelAccountDefault,
21 MVideoWithBlacklistLight
22 } from '@server/types/models'
23 import { MVideoImportFormattable } from '@server/types/models/video/video-import'
32 } from '@shared/models'
33 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
34 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
35 import { isArray } from '../../../helpers/custom-validators/misc'
36 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
37 import { logger } from '../../../helpers/logger'
38 import { getSecureTorrentName } from '../../../helpers/utils'
39 import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
40 import { CONFIG } from '../../../initializers/config'
41 import { MIMETYPES } from '../../../initializers/constants'
42 import { sequelizeTypescript } from '../../../initializers/database'
43 import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
44 import { JobQueue } from '../../../lib/job-queue/job-queue'
45 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
46 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
49 asyncRetryTransactionMiddleware,
51 videoImportAddValidator,
52 videoImportCancelValidator,
53 videoImportDeleteValidator
54 } from '../../../middlewares'
55 import { VideoModel } from '../../../models/video/video'
56 import { VideoCaptionModel } from '../../../models/video/video-caption'
57 import { VideoImportModel } from '../../../models/video/video-import'
59 const auditLogger = auditLoggerFactory('video-imports')
60 const videoImportsRouter = express.Router()
62 const reqVideoFileImport = createReqFiles(
63 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
64 { ...MIMETYPES.TORRENT.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT }
67 videoImportsRouter.post('/imports',
70 asyncMiddleware(videoImportAddValidator),
71 asyncRetryTransactionMiddleware(addVideoImport)
74 videoImportsRouter.post('/imports/:id/cancel',
76 asyncMiddleware(videoImportCancelValidator),
77 asyncRetryTransactionMiddleware(cancelVideoImport)
80 videoImportsRouter.delete('/imports/:id',
82 asyncMiddleware(videoImportDeleteValidator),
83 asyncRetryTransactionMiddleware(deleteVideoImport)
86 // ---------------------------------------------------------------------------
92 // ---------------------------------------------------------------------------
94 async function deleteVideoImport (req: express.Request, res: express.Response) {
95 const videoImport = res.locals.videoImport
97 await videoImport.destroy()
99 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
102 async function cancelVideoImport (req: express.Request, res: express.Response) {
103 const videoImport = res.locals.videoImport
105 videoImport.state = VideoImportState.CANCELLED
106 await videoImport.save()
108 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
111 function addVideoImport (req: express.Request, res: express.Response) {
112 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
114 const file = req.files?.['torrentfile']?.[0]
115 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
118 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
119 const body: VideoImportCreate = req.body
120 const user = res.locals.oauth.token.User
122 let videoName: string
123 let torrentName: string
124 let magnetUri: string
127 const result = await processTorrentOrAbortRequest(req, res, torrentfile)
130 videoName = result.name
131 torrentName = result.torrentName
133 const result = processMagnetURI(body)
134 magnetUri = result.magnetUri
135 videoName = result.name
138 const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
140 const thumbnailModel = await processThumbnail(req, video)
141 const previewModel = await processPreview(req, video)
143 const videoImport = await insertIntoDB({
147 videoChannel: res.locals.videoChannel,
148 tags: body.tags || undefined,
150 videoImportAttributes: {
153 state: VideoImportState.PENDING,
158 // Create job to import the video
161 ? 'torrent-file' as 'torrent-file'
162 : 'magnet-uri' as 'magnet-uri',
163 videoImportId: videoImport.id,
166 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
168 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
170 return res.json(videoImport.toFormattedJSON()).end()
173 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
174 const body: VideoImportCreate = req.body
175 const targetUrl = body.targetUrl
176 const user = res.locals.oauth.token.User
178 const youtubeDL = new YoutubeDLWrapper(
180 ServerConfigManager.Instance.getEnabledResolutions('vod'),
181 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
185 let youtubeDLInfo: YoutubeDLInfo
187 youtubeDLInfo = await youtubeDL.getInfoForDownload()
189 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
192 message: 'Cannot fetch remote information of this URL.',
199 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
201 status: HttpStatusCode.FORBIDDEN_403,
202 message: 'Cannot use non unicast IP as targetUrl.'
206 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
208 // Process video thumbnail from request.files
209 let thumbnailModel = await processThumbnail(req, video)
211 // Process video thumbnail from url if processing from request.files failed
212 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
214 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
216 logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
220 // Process video preview from request.files
221 let previewModel = await processPreview(req, video)
223 // Process video preview from url if processing from request.files failed
224 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
226 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
228 logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
232 const videoImport = await insertIntoDB({
236 videoChannel: res.locals.videoChannel,
237 tags: body.tags || youtubeDLInfo.tags,
239 videoImportAttributes: {
241 state: VideoImportState.PENDING,
246 // Get video subtitles
247 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
249 let fileExt = `.${youtubeDLInfo.ext}`
250 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
252 // Create job to import the video
254 type: 'youtube-dl' as 'youtube-dl',
255 videoImportId: videoImport.id,
258 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
260 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
262 return res.json(videoImport.toFormattedJSON()).end()
265 async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
267 name: body.name || importData.name || 'Unknown name',
269 category: body.category || importData.category,
270 licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
271 language: body.language || importData.language,
272 commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
273 downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
274 waitTranscoding: body.waitTranscoding || false,
275 state: VideoState.TO_IMPORT,
276 nsfw: body.nsfw || importData.nsfw || false,
277 description: body.description || importData.description,
278 support: body.support || null,
279 privacy: body.privacy || VideoPrivacy.PRIVATE,
280 duration: 0, // duration will be set by the import job
282 originallyPublishedAt: body.originallyPublishedAt
283 ? new Date(body.originallyPublishedAt)
284 : importData.originallyPublishedAt
287 videoData = await Hooks.wrapObject(
290 ? 'filter:api.video.import-url.video-attribute.result'
291 : 'filter:api.video.import-torrent.video-attribute.result'
294 const video = new VideoModel(videoData)
295 video.url = getLocalVideoActivityPubUrl(video)
300 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
301 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
302 if (thumbnailField) {
303 const thumbnailPhysicalFile = thumbnailField[0]
305 return updateVideoMiniatureFromExisting({
306 inputPath: thumbnailPhysicalFile.path,
308 type: ThumbnailType.MINIATURE,
309 automaticallyGenerated: false
316 async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> {
317 const previewField = req.files ? req.files['previewfile'] : undefined
319 const previewPhysicalFile = previewField[0]
321 return updateVideoMiniatureFromExisting({
322 inputPath: previewPhysicalFile.path,
324 type: ThumbnailType.PREVIEW,
325 automaticallyGenerated: false
332 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
334 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
336 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
341 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
343 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
345 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
350 async function insertIntoDB (parameters: {
351 video: MVideoThumbnail
352 thumbnailModel: MThumbnail
353 previewModel: MThumbnail
354 videoChannel: MChannelAccountDefault
356 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
358 }): Promise<MVideoImportFormattable> {
359 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
361 const videoImport = await sequelizeTypescript.transaction(async t => {
362 const sequelizeOptions = { transaction: t }
364 // Save video object in database
365 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
366 videoCreated.VideoChannel = videoChannel
368 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
369 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
371 await autoBlacklistVideoIfNeeded({
380 await setVideoTags({ video: videoCreated, tags, transaction: t })
382 // Create video import object in database
383 const videoImport = await VideoImportModel.create(
384 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
386 ) as MVideoImportFormattable
387 videoImport.Video = videoCreated
395 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
396 const torrentName = torrentfile.originalname
398 // Rename the torrent to a secured name
399 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
400 await move(torrentfile.path, newTorrentPath, { overwrite: true })
401 torrentfile.path = newTorrentPath
403 const buf = await readFile(torrentfile.path)
404 const parsedTorrent = parseTorrent(buf) as Instance
406 if (parsedTorrent.files.length !== 1) {
410 type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT,
411 message: 'Torrents with only 1 file are supported.'
417 name: extractNameFromArray(parsedTorrent.name),
422 function processMagnetURI (body: VideoImportCreate) {
423 const magnetUri = body.magnetUri
424 const parsed = decode(magnetUri)
427 name: extractNameFromArray(parsed.name),
432 function extractNameFromArray (name: string | string[]) {
433 return isArray(name) ? name[0] : name
436 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
438 const subtitles = await youtubeDL.getSubtitles()
440 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
442 for (const subtitle of subtitles) {
443 if (!await isVTTFileValid(subtitle.path)) {
444 await remove(subtitle.path)
448 const videoCaption = new VideoCaptionModel({
450 language: subtitle.language,
451 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
454 // Move physical file
455 await moveAndProcessCaptionFile(subtitle, videoCaption)
457 await sequelizeTypescript.transaction(async t => {
458 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
462 logger.warn('Cannot get video subtitles.', { err })
466 async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
467 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
468 const uniqHosts = new Set(hosts)
470 for (const h of uniqHosts) {
471 if (await isResolvingToUnicastOnly(h) !== true) {