1 import * as express from 'express'
2 import * as magnetUtil from 'magnet-uri'
4 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
5 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
6 import { MIMETYPES } from '../../../initializers/constants'
7 import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
8 import { createReqFiles } from '../../../helpers/express-utils'
9 import { logger } from '../../../helpers/logger'
10 import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
11 import { VideoModel } from '../../../models/video/video'
12 import { getVideoActivityPubUrl } from '../../../lib/activitypub'
13 import { TagModel } from '../../../models/video/tag'
14 import { VideoImportModel } from '../../../models/video/video-import'
15 import { JobQueue } from '../../../lib/job-queue/job-queue'
16 import { join } from 'path'
17 import { isArray } from '../../../helpers/custom-validators/misc'
18 import { VideoChannelModel } from '../../../models/video/video-channel'
19 import * as Bluebird from 'bluebird'
20 import * as parseTorrent from 'parse-torrent'
21 import { getSecureTorrentName } from '../../../helpers/utils'
22 import { move, readFile } from 'fs-extra'
23 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
24 import { CONFIG } from '../../../initializers/config'
25 import { sequelizeTypescript } from '../../../initializers/database'
26 import { createVideoMiniatureFromExisting } from '../../../lib/thumbnail'
27 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
28 import { ThumbnailModel } from '../../../models/video/thumbnail'
30 const auditLogger = auditLoggerFactory('video-imports')
31 const videoImportsRouter = express.Router()
33 const reqVideoFileImport = createReqFiles(
34 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
35 Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
37 thumbnailfile: CONFIG.STORAGE.TMP_DIR,
38 previewfile: CONFIG.STORAGE.TMP_DIR,
39 torrentfile: CONFIG.STORAGE.TMP_DIR
43 videoImportsRouter.post('/imports',
46 asyncMiddleware(videoImportAddValidator),
47 asyncRetryTransactionMiddleware(addVideoImport)
50 // ---------------------------------------------------------------------------
56 // ---------------------------------------------------------------------------
58 function addVideoImport (req: express.Request, res: express.Response) {
59 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
61 const file = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
62 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
65 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
66 const body: VideoImportCreate = req.body
67 const user = res.locals.oauth.token.User
70 let torrentName: string
74 torrentName = torrentfile.originalname
76 // Rename the torrent to a secured name
77 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
78 await move(torrentfile.path, newTorrentPath)
79 torrentfile.path = newTorrentPath
81 const buf = await readFile(torrentfile.path)
82 const parsedTorrent = parseTorrent(buf)
84 videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string
86 magnetUri = body.magnetUri
88 const parsed = magnetUtil.decode(magnetUri)
89 videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string
92 const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName })
94 const thumbnailModel = await processThumbnail(req, video)
95 const previewModel = await processPreview(req, video)
97 const tags = body.tags || undefined
98 const videoImportAttributes = {
101 state: VideoImportState.PENDING,
104 const videoImport = await insertIntoDB({
108 videoChannel: res.locals.videoChannel,
110 videoImportAttributes
113 // Create job to import the video
115 type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri',
116 videoImportId: videoImport.id,
119 await JobQueue.Instance.createJob({ type: 'video-import', payload })
121 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
123 return res.json(videoImport.toFormattedJSON()).end()
126 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
127 const body: VideoImportCreate = req.body
128 const targetUrl = body.targetUrl
129 const user = res.locals.oauth.token.User
131 let youtubeDLInfo: YoutubeDLInfo
133 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
135 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
137 return res.status(400).json({
138 error: 'Cannot fetch remote information of this URL.'
142 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
144 const thumbnailModel = await processThumbnail(req, video)
145 const previewModel = await processPreview(req, video)
147 const tags = body.tags || youtubeDLInfo.tags
148 const videoImportAttributes = {
150 state: VideoImportState.PENDING,
153 const videoImport = await insertIntoDB({
157 videoChannel: res.locals.videoChannel,
159 videoImportAttributes
162 // Create job to import the video
164 type: 'youtube-dl' as 'youtube-dl',
165 videoImportId: videoImport.id,
166 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
167 downloadThumbnail: !thumbnailModel,
168 downloadPreview: !previewModel
170 await JobQueue.Instance.createJob({ type: 'video-import', payload })
172 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
174 return res.json(videoImport.toFormattedJSON()).end()
177 function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
179 name: body.name || importData.name || 'Unknown name',
181 category: body.category || importData.category,
182 licence: body.licence || importData.licence,
183 language: body.language || undefined,
184 commentsEnabled: body.commentsEnabled || true,
185 downloadEnabled: body.downloadEnabled || true,
186 waitTranscoding: body.waitTranscoding || false,
187 state: VideoState.TO_IMPORT,
188 nsfw: body.nsfw || importData.nsfw || false,
189 description: body.description || importData.description,
190 support: body.support || null,
191 privacy: body.privacy || VideoPrivacy.PRIVATE,
192 duration: 0, // duration will be set by the import job
193 channelId: channelId,
194 originallyPublishedAt: importData.originallyPublishedAt
196 const video = new VideoModel(videoData)
197 video.url = getVideoActivityPubUrl(video)
202 async function processThumbnail (req: express.Request, video: VideoModel) {
203 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
204 if (thumbnailField) {
205 const thumbnailPhysicalFile = thumbnailField[ 0 ]
207 return createVideoMiniatureFromExisting(thumbnailPhysicalFile.path, video, ThumbnailType.MINIATURE)
213 async function processPreview (req: express.Request, video: VideoModel) {
214 const previewField = req.files ? req.files['previewfile'] : undefined
216 const previewPhysicalFile = previewField[0]
218 return createVideoMiniatureFromExisting(previewPhysicalFile.path, video, ThumbnailType.PREVIEW)
224 function insertIntoDB (parameters: {
226 thumbnailModel: ThumbnailModel,
227 previewModel: ThumbnailModel,
228 videoChannel: VideoChannelModel,
230 videoImportAttributes: Partial<VideoImportModel>
231 }): Bluebird<VideoImportModel> {
232 let { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes } = parameters
234 return sequelizeTypescript.transaction(async t => {
235 const sequelizeOptions = { transaction: t }
237 // Save video object in database
238 const videoCreated = await video.save(sequelizeOptions)
239 videoCreated.VideoChannel = videoChannel
241 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
242 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
244 await autoBlacklistVideoIfNeeded(video, videoChannel.Account.User, t)
246 // Set tags to the video
248 const tagInstances = await TagModel.findOrCreateTags(tags, t)
250 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
251 videoCreated.Tags = tagInstances
253 videoCreated.Tags = []
256 // Create video import object in database
257 const videoImport = await VideoImportModel.create(
258 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
261 videoImport.Video = videoCreated