1 import * as express from 'express'
2 import * as magnetUtil from 'magnet-uri'
4 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
5 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
6 import { CONFIG, MIMETYPES, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers'
7 import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
8 import { createReqFiles } from '../../../helpers/express-utils'
9 import { logger } from '../../../helpers/logger'
10 import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
11 import { VideoModel } from '../../../models/video/video'
12 import { getVideoActivityPubUrl } from '../../../lib/activitypub'
13 import { TagModel } from '../../../models/video/tag'
14 import { VideoImportModel } from '../../../models/video/video-import'
15 import { JobQueue } from '../../../lib/job-queue/job-queue'
16 import { processImage } from '../../../helpers/image-utils'
17 import { join } from 'path'
18 import { isArray } from '../../../helpers/custom-validators/misc'
19 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
20 import { VideoChannelModel } from '../../../models/video/video-channel'
21 import * as Bluebird from 'bluebird'
22 import * as parseTorrent from 'parse-torrent'
23 import { getSecureTorrentName } from '../../../helpers/utils'
24 import { readFile, move } from 'fs-extra'
26 const auditLogger = auditLoggerFactory('video-imports')
27 const videoImportsRouter = express.Router()
29 const reqVideoFileImport = createReqFiles(
30 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
31 Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
33 thumbnailfile: CONFIG.STORAGE.TMP_DIR,
34 previewfile: CONFIG.STORAGE.TMP_DIR,
35 torrentfile: CONFIG.STORAGE.TMP_DIR
39 videoImportsRouter.post('/imports',
42 asyncMiddleware(videoImportAddValidator),
43 asyncRetryTransactionMiddleware(addVideoImport)
46 // ---------------------------------------------------------------------------
52 // ---------------------------------------------------------------------------
54 function addVideoImport (req: express.Request, res: express.Response) {
55 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
57 const file = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
58 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
61 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
62 const body: VideoImportCreate = req.body
63 const user = res.locals.oauth.token.User
66 let torrentName: string
70 torrentName = torrentfile.originalname
72 // Rename the torrent to a secured name
73 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
74 await move(torrentfile.path, newTorrentPath)
75 torrentfile.path = newTorrentPath
77 const buf = await readFile(torrentfile.path)
78 const parsedTorrent = parseTorrent(buf)
80 videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string
82 magnetUri = body.magnetUri
84 const parsed = magnetUtil.decode(magnetUri)
85 videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string
88 const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName })
90 await processThumbnail(req, video)
91 await processPreview(req, video)
93 const tags = body.tags || undefined
94 const videoImportAttributes = {
97 state: VideoImportState.PENDING,
100 const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
102 // Create job to import the video
104 type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri',
105 videoImportId: videoImport.id,
108 await JobQueue.Instance.createJob({ type: 'video-import', payload })
110 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
112 return res.json(videoImport.toFormattedJSON()).end()
115 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
116 const body: VideoImportCreate = req.body
117 const targetUrl = body.targetUrl
118 const user = res.locals.oauth.token.User
120 let youtubeDLInfo: YoutubeDLInfo
122 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
124 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
126 return res.status(400).json({
127 error: 'Cannot fetch remote information of this URL.'
131 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
133 const downloadThumbnail = !await processThumbnail(req, video)
134 const downloadPreview = !await processPreview(req, video)
136 const tags = body.tags || youtubeDLInfo.tags
137 const videoImportAttributes = {
139 state: VideoImportState.PENDING,
142 const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
144 // Create job to import the video
146 type: 'youtube-dl' as 'youtube-dl',
147 videoImportId: videoImport.id,
148 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
152 await JobQueue.Instance.createJob({ type: 'video-import', payload })
154 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
156 return res.json(videoImport.toFormattedJSON()).end()
159 function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
161 name: body.name || importData.name || 'Unknown name',
163 category: body.category || importData.category,
164 licence: body.licence || importData.licence,
165 language: body.language || undefined,
166 commentsEnabled: body.commentsEnabled || true,
167 waitTranscoding: body.waitTranscoding || false,
168 state: VideoState.TO_IMPORT,
169 nsfw: body.nsfw || importData.nsfw || false,
170 description: body.description || importData.description,
171 support: body.support || null,
172 privacy: body.privacy || VideoPrivacy.PRIVATE,
173 duration: 0, // duration will be set by the import job
176 const video = new VideoModel(videoData)
177 video.url = getVideoActivityPubUrl(video)
182 async function processThumbnail (req: express.Request, video: VideoModel) {
183 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
184 if (thumbnailField) {
185 const thumbnailPhysicalFile = thumbnailField[ 0 ]
186 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
194 async function processPreview (req: express.Request, video: VideoModel) {
195 const previewField = req.files ? req.files['previewfile'] : undefined
197 const previewPhysicalFile = previewField[0]
198 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
206 function insertIntoDB (
208 videoChannel: VideoChannelModel,
210 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
211 ): Bluebird<VideoImportModel> {
212 return sequelizeTypescript.transaction(async t => {
213 const sequelizeOptions = { transaction: t }
215 // Save video object in database
216 const videoCreated = await video.save(sequelizeOptions)
217 videoCreated.VideoChannel = videoChannel
219 // Set tags to the video
221 const tagInstances = await TagModel.findOrCreateTags(tags, t)
223 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
224 videoCreated.Tags = tagInstances
226 videoCreated.Tags = []
229 // Create video import object in database
230 const videoImport = await VideoImportModel.create(
231 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
234 videoImport.Video = videoCreated