1 import * as express from 'express'
2 import * as magnetUtil from 'magnet-uri'
4 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
5 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
6 import { MIMETYPES } from '../../../initializers/constants'
7 import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
8 import { createReqFiles } from '../../../helpers/express-utils'
9 import { logger } from '../../../helpers/logger'
10 import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
11 import { VideoModel } from '../../../models/video/video'
12 import { getVideoActivityPubUrl } from '../../../lib/activitypub'
13 import { TagModel } from '../../../models/video/tag'
14 import { VideoImportModel } from '../../../models/video/video-import'
15 import { JobQueue } from '../../../lib/job-queue/job-queue'
16 import { join } from 'path'
17 import { isArray } from '../../../helpers/custom-validators/misc'
18 import * as Bluebird from 'bluebird'
19 import * as parseTorrent from 'parse-torrent'
20 import { getSecureTorrentName } from '../../../helpers/utils'
21 import { move, readFile } from 'fs-extra'
22 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
23 import { CONFIG } from '../../../initializers/config'
24 import { sequelizeTypescript } from '../../../initializers/database'
25 import { createVideoMiniatureFromExisting } from '../../../lib/thumbnail'
26 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
28 MChannelActorAccountDefault,
31 MVideoThumbnailAccountDefault,
32 MVideoWithBlacklistLight
33 } from '@server/typings/models'
34 import { MVideoImport, MVideoImportVideo } from '@server/typings/models/video/video-import'
36 const auditLogger = auditLoggerFactory('video-imports')
37 const videoImportsRouter = express.Router()
39 const reqVideoFileImport = createReqFiles(
40 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
41 Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
43 thumbnailfile: CONFIG.STORAGE.TMP_DIR,
44 previewfile: CONFIG.STORAGE.TMP_DIR,
45 torrentfile: CONFIG.STORAGE.TMP_DIR
49 videoImportsRouter.post('/imports',
52 asyncMiddleware(videoImportAddValidator),
53 asyncRetryTransactionMiddleware(addVideoImport)
56 // ---------------------------------------------------------------------------
62 // ---------------------------------------------------------------------------
64 function addVideoImport (req: express.Request, res: express.Response) {
65 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
67 const file = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
68 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
71 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
72 const body: VideoImportCreate = req.body
73 const user = res.locals.oauth.token.User
76 let torrentName: string
80 torrentName = torrentfile.originalname
82 // Rename the torrent to a secured name
83 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
84 await move(torrentfile.path, newTorrentPath)
85 torrentfile.path = newTorrentPath
87 const buf = await readFile(torrentfile.path)
88 const parsedTorrent = parseTorrent(buf)
90 videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string
92 magnetUri = body.magnetUri
94 const parsed = magnetUtil.decode(magnetUri)
95 videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string
98 const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName })
100 const thumbnailModel = await processThumbnail(req, video)
101 const previewModel = await processPreview(req, video)
103 const tags = body.tags || undefined
104 const videoImportAttributes = {
107 state: VideoImportState.PENDING,
110 const videoImport = await insertIntoDB({
114 videoChannel: res.locals.videoChannel,
116 videoImportAttributes,
120 // Create job to import the video
122 type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri',
123 videoImportId: videoImport.id,
126 await JobQueue.Instance.createJob({ type: 'video-import', payload })
128 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
130 return res.json(videoImport.toFormattedJSON()).end()
133 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
134 const body: VideoImportCreate = req.body
135 const targetUrl = body.targetUrl
136 const user = res.locals.oauth.token.User
138 let youtubeDLInfo: YoutubeDLInfo
140 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
142 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
144 return res.status(400).json({
145 error: 'Cannot fetch remote information of this URL.'
149 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
151 const thumbnailModel = await processThumbnail(req, video)
152 const previewModel = await processPreview(req, video)
154 const tags = body.tags || youtubeDLInfo.tags
155 const videoImportAttributes = {
157 state: VideoImportState.PENDING,
160 const videoImport = await insertIntoDB({
164 videoChannel: res.locals.videoChannel,
166 videoImportAttributes,
170 // Create job to import the video
172 type: 'youtube-dl' as 'youtube-dl',
173 videoImportId: videoImport.id,
174 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
175 downloadThumbnail: !thumbnailModel,
176 downloadPreview: !previewModel
178 await JobQueue.Instance.createJob({ type: 'video-import', payload })
180 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
182 return res.json(videoImport.toFormattedJSON()).end()
185 function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
187 name: body.name || importData.name || 'Unknown name',
189 category: body.category || importData.category,
190 licence: body.licence || importData.licence,
191 language: body.language || undefined,
192 commentsEnabled: body.commentsEnabled || true,
193 downloadEnabled: body.downloadEnabled || true,
194 waitTranscoding: body.waitTranscoding || false,
195 state: VideoState.TO_IMPORT,
196 nsfw: body.nsfw || importData.nsfw || false,
197 description: body.description || importData.description,
198 support: body.support || null,
199 privacy: body.privacy || VideoPrivacy.PRIVATE,
200 duration: 0, // duration will be set by the import job
201 channelId: channelId,
202 originallyPublishedAt: importData.originallyPublishedAt
204 const video = new VideoModel(videoData)
205 video.url = getVideoActivityPubUrl(video)
210 async function processThumbnail (req: express.Request, video: VideoModel) {
211 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
212 if (thumbnailField) {
213 const thumbnailPhysicalFile = thumbnailField[ 0 ]
215 return createVideoMiniatureFromExisting(thumbnailPhysicalFile.path, video, ThumbnailType.MINIATURE, false)
221 async function processPreview (req: express.Request, video: VideoModel) {
222 const previewField = req.files ? req.files['previewfile'] : undefined
224 const previewPhysicalFile = previewField[0]
226 return createVideoMiniatureFromExisting(previewPhysicalFile.path, video, ThumbnailType.PREVIEW, false)
232 function insertIntoDB (parameters: {
233 video: MVideoThumbnailAccountDefault,
234 thumbnailModel: MThumbnail,
235 previewModel: MThumbnail,
236 videoChannel: MChannelActorAccountDefault,
238 videoImportAttributes: Partial<MVideoImport>,
240 }): Bluebird<MVideoImportVideo> {
241 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
243 return sequelizeTypescript.transaction(async t => {
244 const sequelizeOptions = { transaction: t }
246 // Save video object in database
247 const videoCreated = await video.save(sequelizeOptions) as (MVideoThumbnailAccountDefault & MVideoWithBlacklistLight)
248 videoCreated.VideoChannel = videoChannel
250 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
251 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
253 await autoBlacklistVideoIfNeeded({
262 // Set tags to the video
264 const tagInstances = await TagModel.findOrCreateTags(tags, t)
266 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
269 // Create video import object in database
270 const videoImport = await VideoImportModel.create(
271 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
273 ) as MVideoImportVideo
274 videoImport.Video = videoCreated