1 import express from 'express'
2 import { move, readFile } from 'fs-extra'
3 import { decode } from 'magnet-uri'
4 import parseTorrent, { Instance } from 'parse-torrent'
5 import { join } from 'path'
6 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
7 import { Hooks } from '@server/lib/plugins/hooks'
8 import { ServerConfigManager } from '@server/lib/server-config-manager'
9 import { setVideoTags } from '@server/lib/video'
10 import { FilteredModelAttributes } from '@server/types'
12 MChannelAccountDefault,
19 MVideoWithBlacklistLight
20 } from '@server/types/models'
21 import { MVideoImportFormattable } from '@server/types/models/video/video-import'
30 } from '@shared/models'
31 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
32 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
33 import { isArray } from '../../../helpers/custom-validators/misc'
34 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
35 import { logger } from '../../../helpers/logger'
36 import { getSecureTorrentName } from '../../../helpers/utils'
37 import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
38 import { CONFIG } from '../../../initializers/config'
39 import { MIMETYPES } from '../../../initializers/constants'
40 import { sequelizeTypescript } from '../../../initializers/database'
41 import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
42 import { JobQueue } from '../../../lib/job-queue/job-queue'
43 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
44 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
47 asyncRetryTransactionMiddleware,
49 videoImportAddValidator,
50 videoImportCancelValidator,
51 videoImportDeleteValidator
52 } from '../../../middlewares'
53 import { VideoModel } from '../../../models/video/video'
54 import { VideoCaptionModel } from '../../../models/video/video-caption'
55 import { VideoImportModel } from '../../../models/video/video-import'
57 const auditLogger = auditLoggerFactory('video-imports')
58 const videoImportsRouter = express.Router()
60 const reqVideoFileImport = createReqFiles(
61 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
62 Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
64 thumbnailfile: CONFIG.STORAGE.TMP_DIR,
65 previewfile: CONFIG.STORAGE.TMP_DIR,
66 torrentfile: CONFIG.STORAGE.TMP_DIR
70 videoImportsRouter.post('/imports',
73 asyncMiddleware(videoImportAddValidator),
74 asyncRetryTransactionMiddleware(addVideoImport)
77 videoImportsRouter.post('/imports/:id/cancel',
79 asyncMiddleware(videoImportCancelValidator),
80 asyncRetryTransactionMiddleware(cancelVideoImport)
83 videoImportsRouter.delete('/imports/:id',
85 asyncMiddleware(videoImportDeleteValidator),
86 asyncRetryTransactionMiddleware(deleteVideoImport)
89 // ---------------------------------------------------------------------------
95 // ---------------------------------------------------------------------------
97 async function deleteVideoImport (req: express.Request, res: express.Response) {
98 const videoImport = res.locals.videoImport
100 await videoImport.destroy()
102 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
105 async function cancelVideoImport (req: express.Request, res: express.Response) {
106 const videoImport = res.locals.videoImport
108 videoImport.state = VideoImportState.CANCELLED
109 await videoImport.save()
111 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
114 function addVideoImport (req: express.Request, res: express.Response) {
115 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
117 const file = req.files?.['torrentfile']?.[0]
118 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
121 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
122 const body: VideoImportCreate = req.body
123 const user = res.locals.oauth.token.User
125 let videoName: string
126 let torrentName: string
127 let magnetUri: string
130 const result = await processTorrentOrAbortRequest(req, res, torrentfile)
133 videoName = result.name
134 torrentName = result.torrentName
136 const result = processMagnetURI(body)
137 magnetUri = result.magnetUri
138 videoName = result.name
141 const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
143 const thumbnailModel = await processThumbnail(req, video)
144 const previewModel = await processPreview(req, video)
146 const videoImport = await insertIntoDB({
150 videoChannel: res.locals.videoChannel,
151 tags: body.tags || undefined,
153 videoImportAttributes: {
156 state: VideoImportState.PENDING,
161 // Create job to import the video
164 ? 'torrent-file' as 'torrent-file'
165 : 'magnet-uri' as 'magnet-uri',
166 videoImportId: videoImport.id,
169 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
171 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
173 return res.json(videoImport.toFormattedJSON()).end()
176 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
177 const body: VideoImportCreate = req.body
178 const targetUrl = body.targetUrl
179 const user = res.locals.oauth.token.User
181 const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
184 let youtubeDLInfo: YoutubeDLInfo
186 youtubeDLInfo = await youtubeDL.getInfoForDownload()
188 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
191 message: 'Cannot fetch remote information of this URL.',
198 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
200 // Process video thumbnail from request.files
201 let thumbnailModel = await processThumbnail(req, video)
203 // Process video thumbnail from url if processing from request.files failed
204 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
206 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
208 logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
212 // Process video preview from request.files
213 let previewModel = await processPreview(req, video)
215 // Process video preview from url if processing from request.files failed
216 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
218 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
220 logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
224 const videoImport = await insertIntoDB({
228 videoChannel: res.locals.videoChannel,
229 tags: body.tags || youtubeDLInfo.tags,
231 videoImportAttributes: {
233 state: VideoImportState.PENDING,
238 // Get video subtitles
239 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
241 let fileExt = `.${youtubeDLInfo.ext}`
242 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
244 // Create job to import the video
246 type: 'youtube-dl' as 'youtube-dl',
247 videoImportId: videoImport.id,
250 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
252 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
254 return res.json(videoImport.toFormattedJSON()).end()
257 async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
259 name: body.name || importData.name || 'Unknown name',
261 category: body.category || importData.category,
262 licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
263 language: body.language || importData.language,
264 commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
265 downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
266 waitTranscoding: body.waitTranscoding || false,
267 state: VideoState.TO_IMPORT,
268 nsfw: body.nsfw || importData.nsfw || false,
269 description: body.description || importData.description,
270 support: body.support || null,
271 privacy: body.privacy || VideoPrivacy.PRIVATE,
272 duration: 0, // duration will be set by the import job
273 channelId: channelId,
274 originallyPublishedAt: body.originallyPublishedAt
275 ? new Date(body.originallyPublishedAt)
276 : importData.originallyPublishedAt
279 videoData = await Hooks.wrapObject(
282 ? 'filter:api.video.import-url.video-attribute.result'
283 : 'filter:api.video.import-torrent.video-attribute.result'
286 const video = new VideoModel(videoData)
287 video.url = getLocalVideoActivityPubUrl(video)
292 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
293 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
294 if (thumbnailField) {
295 const thumbnailPhysicalFile = thumbnailField[0]
297 return updateVideoMiniatureFromExisting({
298 inputPath: thumbnailPhysicalFile.path,
300 type: ThumbnailType.MINIATURE,
301 automaticallyGenerated: false
308 async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> {
309 const previewField = req.files ? req.files['previewfile'] : undefined
311 const previewPhysicalFile = previewField[0]
313 return updateVideoMiniatureFromExisting({
314 inputPath: previewPhysicalFile.path,
316 type: ThumbnailType.PREVIEW,
317 automaticallyGenerated: false
324 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
326 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
328 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
333 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
335 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
337 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
342 async function insertIntoDB (parameters: {
343 video: MVideoThumbnail
344 thumbnailModel: MThumbnail
345 previewModel: MThumbnail
346 videoChannel: MChannelAccountDefault
348 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
350 }): Promise<MVideoImportFormattable> {
351 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
353 const videoImport = await sequelizeTypescript.transaction(async t => {
354 const sequelizeOptions = { transaction: t }
356 // Save video object in database
357 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
358 videoCreated.VideoChannel = videoChannel
360 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
361 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
363 await autoBlacklistVideoIfNeeded({
372 await setVideoTags({ video: videoCreated, tags, transaction: t })
374 // Create video import object in database
375 const videoImport = await VideoImportModel.create(
376 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
378 ) as MVideoImportFormattable
379 videoImport.Video = videoCreated
387 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
388 const torrentName = torrentfile.originalname
390 // Rename the torrent to a secured name
391 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
392 await move(torrentfile.path, newTorrentPath, { overwrite: true })
393 torrentfile.path = newTorrentPath
395 const buf = await readFile(torrentfile.path)
396 const parsedTorrent = parseTorrent(buf) as Instance
398 if (parsedTorrent.files.length !== 1) {
402 type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT,
403 message: 'Torrents with only 1 file are supported.'
409 name: extractNameFromArray(parsedTorrent.name),
414 function processMagnetURI (body: VideoImportCreate) {
415 const magnetUri = body.magnetUri
416 const parsed = decode(magnetUri)
419 name: extractNameFromArray(parsed.name),
424 function extractNameFromArray (name: string | string[]) {
425 return isArray(name) ? name[0] : name
428 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
430 const subtitles = await youtubeDL.getSubtitles()
432 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
434 for (const subtitle of subtitles) {
435 const videoCaption = new VideoCaptionModel({
437 language: subtitle.language,
438 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
441 // Move physical file
442 await moveAndProcessCaptionFile(subtitle, videoCaption)
444 await sequelizeTypescript.transaction(async t => {
445 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
449 logger.warn('Cannot get video subtitles.', { err })