1 import express from 'express'
2 import { move, readFile, remove } from 'fs-extra'
3 import { decode } from 'magnet-uri'
4 import parseTorrent, { Instance } from 'parse-torrent'
5 import { join } from 'path'
6 import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
7 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
8 import { isResolvingToUnicastOnly } from '@server/helpers/dns'
9 import { Hooks } from '@server/lib/plugins/hooks'
10 import { ServerConfigManager } from '@server/lib/server-config-manager'
11 import { setVideoTags } from '@server/lib/video'
12 import { FilteredModelAttributes } from '@server/types'
14 MChannelAccountDefault,
21 MVideoWithBlacklistLight
22 } from '@server/types/models'
23 import { MVideoImportFormattable } from '@server/types/models/video/video-import'
32 } from '@shared/models'
33 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
34 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
35 import { isArray } from '../../../helpers/custom-validators/misc'
36 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
37 import { logger } from '../../../helpers/logger'
38 import { getSecureTorrentName } from '../../../helpers/utils'
39 import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
40 import { CONFIG } from '../../../initializers/config'
41 import { MIMETYPES } from '../../../initializers/constants'
42 import { sequelizeTypescript } from '../../../initializers/database'
43 import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
44 import { JobQueue } from '../../../lib/job-queue/job-queue'
45 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
46 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
49 asyncRetryTransactionMiddleware,
51 videoImportAddValidator,
52 videoImportCancelValidator,
53 videoImportDeleteValidator
54 } from '../../../middlewares'
55 import { VideoModel } from '../../../models/video/video'
56 import { VideoCaptionModel } from '../../../models/video/video-caption'
57 import { VideoImportModel } from '../../../models/video/video-import'
59 const auditLogger = auditLoggerFactory('video-imports')
60 const videoImportsRouter = express.Router()
62 const reqVideoFileImport = createReqFiles(
63 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
64 { ...MIMETYPES.TORRENT.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT }
67 videoImportsRouter.post('/imports',
70 asyncMiddleware(videoImportAddValidator),
71 asyncRetryTransactionMiddleware(addVideoImport)
74 videoImportsRouter.post('/imports/:id/cancel',
76 asyncMiddleware(videoImportCancelValidator),
77 asyncRetryTransactionMiddleware(cancelVideoImport)
80 videoImportsRouter.delete('/imports/:id',
82 asyncMiddleware(videoImportDeleteValidator),
83 asyncRetryTransactionMiddleware(deleteVideoImport)
86 // ---------------------------------------------------------------------------
92 // ---------------------------------------------------------------------------
94 async function deleteVideoImport (req: express.Request, res: express.Response) {
95 const videoImport = res.locals.videoImport
97 await videoImport.destroy()
99 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
102 async function cancelVideoImport (req: express.Request, res: express.Response) {
103 const videoImport = res.locals.videoImport
105 videoImport.state = VideoImportState.CANCELLED
106 await videoImport.save()
108 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
111 function addVideoImport (req: express.Request, res: express.Response) {
112 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
114 const file = req.files?.['torrentfile']?.[0]
115 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
118 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
119 const body: VideoImportCreate = req.body
120 const user = res.locals.oauth.token.User
122 let videoName: string
123 let torrentName: string
124 let magnetUri: string
127 const result = await processTorrentOrAbortRequest(req, res, torrentfile)
130 videoName = result.name
131 torrentName = result.torrentName
133 const result = processMagnetURI(body)
134 magnetUri = result.magnetUri
135 videoName = result.name
138 const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
140 const thumbnailModel = await processThumbnail(req, video)
141 const previewModel = await processPreview(req, video)
143 const videoImport = await insertIntoDB({
147 videoChannel: res.locals.videoChannel,
148 tags: body.tags || undefined,
150 videoImportAttributes: {
153 state: VideoImportState.PENDING,
158 // Create job to import the video
161 ? 'torrent-file' as 'torrent-file'
162 : 'magnet-uri' as 'magnet-uri',
163 videoImportId: videoImport.id,
166 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
168 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
170 return res.json(videoImport.toFormattedJSON()).end()
173 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
174 const body: VideoImportCreate = req.body
175 const targetUrl = body.targetUrl
176 const user = res.locals.oauth.token.User
178 const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
181 let youtubeDLInfo: YoutubeDLInfo
183 youtubeDLInfo = await youtubeDL.getInfoForDownload()
185 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
188 message: 'Cannot fetch remote information of this URL.',
195 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
197 status: HttpStatusCode.FORBIDDEN_403,
198 message: 'Cannot use non unicast IP as targetUrl.'
202 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
204 // Process video thumbnail from request.files
205 let thumbnailModel = await processThumbnail(req, video)
207 // Process video thumbnail from url if processing from request.files failed
208 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
210 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
212 logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
216 // Process video preview from request.files
217 let previewModel = await processPreview(req, video)
219 // Process video preview from url if processing from request.files failed
220 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
222 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
224 logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
228 const videoImport = await insertIntoDB({
232 videoChannel: res.locals.videoChannel,
233 tags: body.tags || youtubeDLInfo.tags,
235 videoImportAttributes: {
237 state: VideoImportState.PENDING,
242 // Get video subtitles
243 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
245 let fileExt = `.${youtubeDLInfo.ext}`
246 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
248 // Create job to import the video
250 type: 'youtube-dl' as 'youtube-dl',
251 videoImportId: videoImport.id,
254 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
256 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
258 return res.json(videoImport.toFormattedJSON()).end()
261 async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
263 name: body.name || importData.name || 'Unknown name',
265 category: body.category || importData.category,
266 licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
267 language: body.language || importData.language,
268 commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
269 downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
270 waitTranscoding: body.waitTranscoding || false,
271 state: VideoState.TO_IMPORT,
272 nsfw: body.nsfw || importData.nsfw || false,
273 description: body.description || importData.description,
274 support: body.support || null,
275 privacy: body.privacy || VideoPrivacy.PRIVATE,
276 duration: 0, // duration will be set by the import job
277 channelId: channelId,
278 originallyPublishedAt: body.originallyPublishedAt
279 ? new Date(body.originallyPublishedAt)
280 : importData.originallyPublishedAt
283 videoData = await Hooks.wrapObject(
286 ? 'filter:api.video.import-url.video-attribute.result'
287 : 'filter:api.video.import-torrent.video-attribute.result'
290 const video = new VideoModel(videoData)
291 video.url = getLocalVideoActivityPubUrl(video)
296 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
297 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
298 if (thumbnailField) {
299 const thumbnailPhysicalFile = thumbnailField[0]
301 return updateVideoMiniatureFromExisting({
302 inputPath: thumbnailPhysicalFile.path,
304 type: ThumbnailType.MINIATURE,
305 automaticallyGenerated: false
312 async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> {
313 const previewField = req.files ? req.files['previewfile'] : undefined
315 const previewPhysicalFile = previewField[0]
317 return updateVideoMiniatureFromExisting({
318 inputPath: previewPhysicalFile.path,
320 type: ThumbnailType.PREVIEW,
321 automaticallyGenerated: false
328 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
330 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
332 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
337 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
339 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
341 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
346 async function insertIntoDB (parameters: {
347 video: MVideoThumbnail
348 thumbnailModel: MThumbnail
349 previewModel: MThumbnail
350 videoChannel: MChannelAccountDefault
352 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
354 }): Promise<MVideoImportFormattable> {
355 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
357 const videoImport = await sequelizeTypescript.transaction(async t => {
358 const sequelizeOptions = { transaction: t }
360 // Save video object in database
361 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
362 videoCreated.VideoChannel = videoChannel
364 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
365 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
367 await autoBlacklistVideoIfNeeded({
376 await setVideoTags({ video: videoCreated, tags, transaction: t })
378 // Create video import object in database
379 const videoImport = await VideoImportModel.create(
380 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
382 ) as MVideoImportFormattable
383 videoImport.Video = videoCreated
391 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
392 const torrentName = torrentfile.originalname
394 // Rename the torrent to a secured name
395 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
396 await move(torrentfile.path, newTorrentPath, { overwrite: true })
397 torrentfile.path = newTorrentPath
399 const buf = await readFile(torrentfile.path)
400 const parsedTorrent = parseTorrent(buf) as Instance
402 if (parsedTorrent.files.length !== 1) {
406 type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT,
407 message: 'Torrents with only 1 file are supported.'
413 name: extractNameFromArray(parsedTorrent.name),
418 function processMagnetURI (body: VideoImportCreate) {
419 const magnetUri = body.magnetUri
420 const parsed = decode(magnetUri)
423 name: extractNameFromArray(parsed.name),
428 function extractNameFromArray (name: string | string[]) {
429 return isArray(name) ? name[0] : name
432 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
434 const subtitles = await youtubeDL.getSubtitles()
436 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
438 for (const subtitle of subtitles) {
439 if (!await isVTTFileValid(subtitle.path)) {
440 await remove(subtitle.path)
444 const videoCaption = new VideoCaptionModel({
446 language: subtitle.language,
447 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
450 // Move physical file
451 await moveAndProcessCaptionFile(subtitle, videoCaption)
453 await sequelizeTypescript.transaction(async t => {
454 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
458 logger.warn('Cannot get video subtitles.', { err })
462 async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
463 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
464 const uniqHosts = new Set(hosts)
466 for (const h of uniqHosts) {
467 if (await isResolvingToUnicastOnly(h) !== true) {