1 import * as express from 'express'
2 import * as magnetUtil from 'magnet-uri'
4 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
5 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
6 import { CONFIG, MIMETYPES, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers'
7 import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
8 import { createReqFiles } from '../../../helpers/express-utils'
9 import { logger } from '../../../helpers/logger'
10 import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
11 import { VideoModel } from '../../../models/video/video'
12 import { getVideoActivityPubUrl } from '../../../lib/activitypub'
13 import { TagModel } from '../../../models/video/tag'
14 import { VideoImportModel } from '../../../models/video/video-import'
15 import { JobQueue } from '../../../lib/job-queue/job-queue'
16 import { processImage } from '../../../helpers/image-utils'
17 import { join } from 'path'
18 import { isArray } from '../../../helpers/custom-validators/misc'
19 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
20 import { VideoChannelModel } from '../../../models/video/video-channel'
21 import { UserModel } from '../../../models/account/user'
22 import * as Bluebird from 'bluebird'
23 import * as parseTorrent from 'parse-torrent'
24 import { getSecureTorrentName } from '../../../helpers/utils'
25 import { readFile, move } from 'fs-extra'
26 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
28 const auditLogger = auditLoggerFactory('video-imports')
29 const videoImportsRouter = express.Router()
31 const reqVideoFileImport = createReqFiles(
32 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
33 Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
35 thumbnailfile: CONFIG.STORAGE.TMP_DIR,
36 previewfile: CONFIG.STORAGE.TMP_DIR,
37 torrentfile: CONFIG.STORAGE.TMP_DIR
41 videoImportsRouter.post('/imports',
44 asyncMiddleware(videoImportAddValidator),
45 asyncRetryTransactionMiddleware(addVideoImport)
48 // ---------------------------------------------------------------------------
54 // ---------------------------------------------------------------------------
56 function addVideoImport (req: express.Request, res: express.Response) {
57 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
59 const file = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
60 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
63 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
64 const body: VideoImportCreate = req.body
65 const user = res.locals.oauth.token.User
68 let torrentName: string
72 torrentName = torrentfile.originalname
74 // Rename the torrent to a secured name
75 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
76 await move(torrentfile.path, newTorrentPath)
77 torrentfile.path = newTorrentPath
79 const buf = await readFile(torrentfile.path)
80 const parsedTorrent = parseTorrent(buf)
82 videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string
84 magnetUri = body.magnetUri
86 const parsed = magnetUtil.decode(magnetUri)
87 videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string
90 const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName }, user)
92 await processThumbnail(req, video)
93 await processPreview(req, video)
95 const tags = body.tags || undefined
96 const videoImportAttributes = {
99 state: VideoImportState.PENDING,
102 const videoImport = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
104 // Create job to import the video
106 type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri',
107 videoImportId: videoImport.id,
110 await JobQueue.Instance.createJob({ type: 'video-import', payload })
112 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
114 return res.json(videoImport.toFormattedJSON()).end()
117 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
118 const body: VideoImportCreate = req.body
119 const targetUrl = body.targetUrl
120 const user = res.locals.oauth.token.User
122 let youtubeDLInfo: YoutubeDLInfo
124 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
126 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
128 return res.status(400).json({
129 error: 'Cannot fetch remote information of this URL.'
133 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo, user)
135 const downloadThumbnail = !await processThumbnail(req, video)
136 const downloadPreview = !await processPreview(req, video)
138 const tags = body.tags || youtubeDLInfo.tags
139 const videoImportAttributes = {
141 state: VideoImportState.PENDING,
144 const videoImport = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
146 // Create job to import the video
148 type: 'youtube-dl' as 'youtube-dl',
149 videoImportId: videoImport.id,
150 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
154 await JobQueue.Instance.createJob({ type: 'video-import', payload })
156 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
158 return res.json(videoImport.toFormattedJSON()).end()
161 function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo, user: UserModel) {
163 name: body.name || importData.name || 'Unknown name',
165 category: body.category || importData.category,
166 licence: body.licence || importData.licence,
167 language: body.language || undefined,
168 commentsEnabled: body.commentsEnabled || true,
169 downloadEnabled: body.downloadEnabled || true,
170 waitTranscoding: body.waitTranscoding || false,
171 state: VideoState.TO_IMPORT,
172 nsfw: body.nsfw || importData.nsfw || false,
173 description: body.description || importData.description,
174 support: body.support || null,
175 privacy: body.privacy || VideoPrivacy.PRIVATE,
176 duration: 0, // duration will be set by the import job
177 channelId: channelId,
178 originallyPublishedAt: importData.originallyPublishedAt
180 const video = new VideoModel(videoData)
181 video.url = getVideoActivityPubUrl(video)
186 async function processThumbnail (req: express.Request, video: VideoModel) {
187 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
188 if (thumbnailField) {
189 const thumbnailPhysicalFile = thumbnailField[ 0 ]
190 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
198 async function processPreview (req: express.Request, video: VideoModel) {
199 const previewField = req.files ? req.files['previewfile'] : undefined
201 const previewPhysicalFile = previewField[0]
202 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
210 function insertIntoDB (
212 videoChannel: VideoChannelModel,
214 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
215 ): Bluebird<VideoImportModel> {
216 return sequelizeTypescript.transaction(async t => {
217 const sequelizeOptions = { transaction: t }
219 // Save video object in database
220 const videoCreated = await video.save(sequelizeOptions)
221 videoCreated.VideoChannel = videoChannel
223 await autoBlacklistVideoIfNeeded(video, videoChannel.Account.User, t)
225 // Set tags to the video
227 const tagInstances = await TagModel.findOrCreateTags(tags, t)
229 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
230 videoCreated.Tags = tagInstances
232 videoCreated.Tags = []
235 // Create video import object in database
236 const videoImport = await VideoImportModel.create(
237 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
240 videoImport.Video = videoCreated