1 import express from 'express'
2 import { move, readFile } from 'fs-extra'
3 import { decode } from 'magnet-uri'
4 import parseTorrent, { Instance } from 'parse-torrent'
5 import { join } from 'path'
6 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
7 import { ServerConfigManager } from '@server/lib/server-config-manager'
8 import { setVideoTags } from '@server/lib/video'
9 import { FilteredModelAttributes } from '@server/types'
11 MChannelAccountDefault,
18 MVideoWithBlacklistLight
19 } from '@server/types/models'
20 import { MVideoImportFormattable } from '@server/types/models/video/video-import'
21 import { ServerErrorCode, VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
22 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
23 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
24 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
25 import { isArray } from '../../../helpers/custom-validators/misc'
26 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
27 import { logger } from '../../../helpers/logger'
28 import { getSecureTorrentName } from '../../../helpers/utils'
29 import { YoutubeDLWrapper, YoutubeDLInfo } from '../../../helpers/youtube-dl'
30 import { CONFIG } from '../../../initializers/config'
31 import { MIMETYPES } from '../../../initializers/constants'
32 import { sequelizeTypescript } from '../../../initializers/database'
33 import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
34 import { JobQueue } from '../../../lib/job-queue/job-queue'
35 import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
36 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
37 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
38 import { VideoModel } from '../../../models/video/video'
39 import { VideoCaptionModel } from '../../../models/video/video-caption'
40 import { VideoImportModel } from '../../../models/video/video-import'
41 import { Hooks } from '@server/lib/plugins/hooks'
43 const auditLogger = auditLoggerFactory('video-imports')
44 const videoImportsRouter = express.Router()
46 const reqVideoFileImport = createReqFiles(
47 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
48 Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
50 thumbnailfile: CONFIG.STORAGE.TMP_DIR,
51 previewfile: CONFIG.STORAGE.TMP_DIR,
52 torrentfile: CONFIG.STORAGE.TMP_DIR
56 videoImportsRouter.post('/imports',
59 asyncMiddleware(videoImportAddValidator),
60 asyncRetryTransactionMiddleware(addVideoImport)
63 // ---------------------------------------------------------------------------
69 // ---------------------------------------------------------------------------
71 function addVideoImport (req: express.Request, res: express.Response) {
72 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
74 const file = req.files?.['torrentfile']?.[0]
75 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
78 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
79 const body: VideoImportCreate = req.body
80 const user = res.locals.oauth.token.User
83 let torrentName: string
87 const result = await processTorrentOrAbortRequest(req, res, torrentfile)
90 videoName = result.name
91 torrentName = result.torrentName
93 const result = processMagnetURI(body)
94 magnetUri = result.magnetUri
95 videoName = result.name
98 const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
100 const thumbnailModel = await processThumbnail(req, video)
101 const previewModel = await processPreview(req, video)
103 const videoImport = await insertIntoDB({
107 videoChannel: res.locals.videoChannel,
108 tags: body.tags || undefined,
110 videoImportAttributes: {
113 state: VideoImportState.PENDING,
118 // Create job to import the video
121 ? 'torrent-file' as 'torrent-file'
122 : 'magnet-uri' as 'magnet-uri',
123 videoImportId: videoImport.id,
126 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
128 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
130 return res.json(videoImport.toFormattedJSON()).end()
133 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
134 const body: VideoImportCreate = req.body
135 const targetUrl = body.targetUrl
136 const user = res.locals.oauth.token.User
138 const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
141 let youtubeDLInfo: YoutubeDLInfo
143 youtubeDLInfo = await youtubeDL.getInfoForDownload()
145 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
148 message: 'Cannot fetch remote information of this URL.',
155 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
157 // Process video thumbnail from request.files
158 let thumbnailModel = await processThumbnail(req, video)
160 // Process video thumbnail from url if processing from request.files failed
161 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
163 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
165 logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
169 // Process video preview from request.files
170 let previewModel = await processPreview(req, video)
172 // Process video preview from url if processing from request.files failed
173 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
175 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
177 logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
181 const videoImport = await insertIntoDB({
185 videoChannel: res.locals.videoChannel,
186 tags: body.tags || youtubeDLInfo.tags,
188 videoImportAttributes: {
190 state: VideoImportState.PENDING,
195 // Get video subtitles
196 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
198 let fileExt = `.${youtubeDLInfo.ext}`
199 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
201 // Create job to import the video
203 type: 'youtube-dl' as 'youtube-dl',
204 videoImportId: videoImport.id,
207 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
209 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
211 return res.json(videoImport.toFormattedJSON()).end()
214 async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
216 name: body.name || importData.name || 'Unknown name',
218 category: body.category || importData.category,
219 licence: body.licence || importData.licence,
220 language: body.language || importData.language,
221 commentsEnabled: body.commentsEnabled !== false, // If the value is not "false", the default is "true"
222 downloadEnabled: body.downloadEnabled !== false,
223 waitTranscoding: body.waitTranscoding || false,
224 state: VideoState.TO_IMPORT,
225 nsfw: body.nsfw || importData.nsfw || false,
226 description: body.description || importData.description,
227 support: body.support || null,
228 privacy: body.privacy || VideoPrivacy.PRIVATE,
229 duration: 0, // duration will be set by the import job
230 channelId: channelId,
231 originallyPublishedAt: body.originallyPublishedAt
232 ? new Date(body.originallyPublishedAt)
233 : importData.originallyPublishedAt
236 videoData = await Hooks.wrapObject(
239 ? 'filter:api.video.import-url.video-attribute.result'
240 : 'filter:api.video.import-torrent.video-attribute.result'
243 const video = new VideoModel(videoData)
244 video.url = getLocalVideoActivityPubUrl(video)
249 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
250 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
251 if (thumbnailField) {
252 const thumbnailPhysicalFile = thumbnailField[0]
254 return updateVideoMiniatureFromExisting({
255 inputPath: thumbnailPhysicalFile.path,
257 type: ThumbnailType.MINIATURE,
258 automaticallyGenerated: false
265 async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> {
266 const previewField = req.files ? req.files['previewfile'] : undefined
268 const previewPhysicalFile = previewField[0]
270 return updateVideoMiniatureFromExisting({
271 inputPath: previewPhysicalFile.path,
273 type: ThumbnailType.PREVIEW,
274 automaticallyGenerated: false
281 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
283 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
285 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
290 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
292 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
294 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
299 async function insertIntoDB (parameters: {
300 video: MVideoThumbnail
301 thumbnailModel: MThumbnail
302 previewModel: MThumbnail
303 videoChannel: MChannelAccountDefault
305 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
307 }): Promise<MVideoImportFormattable> {
308 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
310 const videoImport = await sequelizeTypescript.transaction(async t => {
311 const sequelizeOptions = { transaction: t }
313 // Save video object in database
314 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
315 videoCreated.VideoChannel = videoChannel
317 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
318 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
320 await autoBlacklistVideoIfNeeded({
329 await setVideoTags({ video: videoCreated, tags, transaction: t })
331 // Create video import object in database
332 const videoImport = await VideoImportModel.create(
333 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
335 ) as MVideoImportFormattable
336 videoImport.Video = videoCreated
344 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
345 const torrentName = torrentfile.originalname
347 // Rename the torrent to a secured name
348 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
349 await move(torrentfile.path, newTorrentPath, { overwrite: true })
350 torrentfile.path = newTorrentPath
352 const buf = await readFile(torrentfile.path)
353 const parsedTorrent = parseTorrent(buf) as Instance
355 if (parsedTorrent.files.length !== 1) {
359 type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT,
360 message: 'Torrents with only 1 file are supported.'
366 name: extractNameFromArray(parsedTorrent.name),
371 function processMagnetURI (body: VideoImportCreate) {
372 const magnetUri = body.magnetUri
373 const parsed = decode(magnetUri)
376 name: extractNameFromArray(parsed.name),
381 function extractNameFromArray (name: string | string[]) {
382 return isArray(name) ? name[0] : name
385 async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
387 const subtitles = await youtubeDL.getSubtitles()
389 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
391 for (const subtitle of subtitles) {
392 const videoCaption = new VideoCaptionModel({
394 language: subtitle.language,
395 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
398 // Move physical file
399 await moveAndProcessCaptionFile(subtitle, videoCaption)
401 await sequelizeTypescript.transaction(async t => {
402 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
406 logger.warn('Cannot get video subtitles.', { err })