1 import * as express from 'express'
2 import * as magnetUtil from 'magnet-uri'
4 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
5 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
6 import { MIMETYPES, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../../../initializers/constants'
7 import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
8 import { createReqFiles } from '../../../helpers/express-utils'
9 import { logger } from '../../../helpers/logger'
10 import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
11 import { VideoModel } from '../../../models/video/video'
12 import { getVideoActivityPubUrl } from '../../../lib/activitypub'
13 import { TagModel } from '../../../models/video/tag'
14 import { VideoImportModel } from '../../../models/video/video-import'
15 import { JobQueue } from '../../../lib/job-queue/job-queue'
16 import { processImage } from '../../../helpers/image-utils'
17 import { join } from 'path'
18 import { isArray } from '../../../helpers/custom-validators/misc'
19 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
20 import { VideoChannelModel } from '../../../models/video/video-channel'
21 import { UserModel } from '../../../models/account/user'
22 import * as Bluebird from 'bluebird'
23 import * as parseTorrent from 'parse-torrent'
24 import { getSecureTorrentName } from '../../../helpers/utils'
25 import { move, readFile } from 'fs-extra'
26 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
27 import { CONFIG } from '../../../initializers/config'
28 import { sequelizeTypescript } from '../../../initializers/database'
30 const auditLogger = auditLoggerFactory('video-imports')
31 const videoImportsRouter = express.Router()
33 const reqVideoFileImport = createReqFiles(
34 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
35 Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
37 thumbnailfile: CONFIG.STORAGE.TMP_DIR,
38 previewfile: CONFIG.STORAGE.TMP_DIR,
39 torrentfile: CONFIG.STORAGE.TMP_DIR
43 videoImportsRouter.post('/imports',
46 asyncMiddleware(videoImportAddValidator),
47 asyncRetryTransactionMiddleware(addVideoImport)
50 // ---------------------------------------------------------------------------
56 // ---------------------------------------------------------------------------
58 function addVideoImport (req: express.Request, res: express.Response) {
59 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
61 const file = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
62 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
65 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
66 const body: VideoImportCreate = req.body
67 const user = res.locals.oauth.token.User
70 let torrentName: string
74 torrentName = torrentfile.originalname
76 // Rename the torrent to a secured name
77 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
78 await move(torrentfile.path, newTorrentPath)
79 torrentfile.path = newTorrentPath
81 const buf = await readFile(torrentfile.path)
82 const parsedTorrent = parseTorrent(buf)
84 videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string
86 magnetUri = body.magnetUri
88 const parsed = magnetUtil.decode(magnetUri)
89 videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string
92 const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName }, user)
94 await processThumbnail(req, video)
95 await processPreview(req, video)
97 const tags = body.tags || undefined
98 const videoImportAttributes = {
101 state: VideoImportState.PENDING,
104 const videoImport = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
106 // Create job to import the video
108 type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri',
109 videoImportId: videoImport.id,
112 await JobQueue.Instance.createJob({ type: 'video-import', payload })
114 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
116 return res.json(videoImport.toFormattedJSON()).end()
119 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
120 const body: VideoImportCreate = req.body
121 const targetUrl = body.targetUrl
122 const user = res.locals.oauth.token.User
124 let youtubeDLInfo: YoutubeDLInfo
126 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
128 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
130 return res.status(400).json({
131 error: 'Cannot fetch remote information of this URL.'
135 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo, user)
137 const downloadThumbnail = !await processThumbnail(req, video)
138 const downloadPreview = !await processPreview(req, video)
140 const tags = body.tags || youtubeDLInfo.tags
141 const videoImportAttributes = {
143 state: VideoImportState.PENDING,
146 const videoImport = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
148 // Create job to import the video
150 type: 'youtube-dl' as 'youtube-dl',
151 videoImportId: videoImport.id,
152 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
156 await JobQueue.Instance.createJob({ type: 'video-import', payload })
158 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
160 return res.json(videoImport.toFormattedJSON()).end()
163 function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo, user: UserModel) {
165 name: body.name || importData.name || 'Unknown name',
167 category: body.category || importData.category,
168 licence: body.licence || importData.licence,
169 language: body.language || undefined,
170 commentsEnabled: body.commentsEnabled || true,
171 downloadEnabled: body.downloadEnabled || true,
172 waitTranscoding: body.waitTranscoding || false,
173 state: VideoState.TO_IMPORT,
174 nsfw: body.nsfw || importData.nsfw || false,
175 description: body.description || importData.description,
176 support: body.support || null,
177 privacy: body.privacy || VideoPrivacy.PRIVATE,
178 duration: 0, // duration will be set by the import job
179 channelId: channelId,
180 originallyPublishedAt: importData.originallyPublishedAt
182 const video = new VideoModel(videoData)
183 video.url = getVideoActivityPubUrl(video)
188 async function processThumbnail (req: express.Request, video: VideoModel) {
189 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
190 if (thumbnailField) {
191 const thumbnailPhysicalFile = thumbnailField[ 0 ]
192 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
200 async function processPreview (req: express.Request, video: VideoModel) {
201 const previewField = req.files ? req.files['previewfile'] : undefined
203 const previewPhysicalFile = previewField[0]
204 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
212 function insertIntoDB (
214 videoChannel: VideoChannelModel,
216 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
217 ): Bluebird<VideoImportModel> {
218 return sequelizeTypescript.transaction(async t => {
219 const sequelizeOptions = { transaction: t }
221 // Save video object in database
222 const videoCreated = await video.save(sequelizeOptions)
223 videoCreated.VideoChannel = videoChannel
225 await autoBlacklistVideoIfNeeded(video, videoChannel.Account.User, t)
227 // Set tags to the video
229 const tagInstances = await TagModel.findOrCreateTags(tags, t)
231 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
232 videoCreated.Tags = tagInstances
234 videoCreated.Tags = []
237 // Create video import object in database
238 const videoImport = await VideoImportModel.create(
239 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
242 videoImport.Video = videoCreated