]>
Commit | Line | Data |
---|---|---|
1 | import express from 'express' | |
2 | import { move, readFile, remove } from 'fs-extra' | |
3 | import { decode } from 'magnet-uri' | |
4 | import parseTorrent, { Instance } from 'parse-torrent' | |
5 | import { join } from 'path' | |
6 | import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' | |
7 | import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' | |
8 | import { isResolvingToUnicastOnly } from '@server/helpers/dns' | |
9 | import { Hooks } from '@server/lib/plugins/hooks' | |
10 | import { ServerConfigManager } from '@server/lib/server-config-manager' | |
11 | import { setVideoTags } from '@server/lib/video' | |
12 | import { FilteredModelAttributes } from '@server/types' | |
13 | import { | |
14 | MChannelAccountDefault, | |
15 | MThumbnail, | |
16 | MUser, | |
17 | MVideoAccountDefault, | |
18 | MVideoCaption, | |
19 | MVideoTag, | |
20 | MVideoThumbnail, | |
21 | MVideoWithBlacklistLight | |
22 | } from '@server/types/models' | |
23 | import { MVideoImportFormattable } from '@server/types/models/video/video-import' | |
24 | import { | |
25 | HttpStatusCode, | |
26 | ServerErrorCode, | |
27 | ThumbnailType, | |
28 | VideoImportCreate, | |
29 | VideoImportState, | |
30 | VideoPrivacy, | |
31 | VideoState | |
32 | } from '@shared/models' | |
33 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' | |
34 | import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' | |
35 | import { isArray } from '../../../helpers/custom-validators/misc' | |
36 | import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' | |
37 | import { logger } from '../../../helpers/logger' | |
38 | import { getSecureTorrentName } from '../../../helpers/utils' | |
39 | import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl' | |
40 | import { CONFIG } from '../../../initializers/config' | |
41 | import { MIMETYPES } from '../../../initializers/constants' | |
42 | import { sequelizeTypescript } from '../../../initializers/database' | |
43 | import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url' | |
44 | import { JobQueue } from '../../../lib/job-queue/job-queue' | |
45 | import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail' | |
46 | import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' | |
47 | import { | |
48 | asyncMiddleware, | |
49 | asyncRetryTransactionMiddleware, | |
50 | authenticate, | |
51 | videoImportAddValidator, | |
52 | videoImportCancelValidator, | |
53 | videoImportDeleteValidator | |
54 | } from '../../../middlewares' | |
55 | import { VideoModel } from '../../../models/video/video' | |
56 | import { VideoCaptionModel } from '../../../models/video/video-caption' | |
57 | import { VideoImportModel } from '../../../models/video/video-import' | |
58 | ||
59 | const auditLogger = auditLoggerFactory('video-imports') | |
60 | const videoImportsRouter = express.Router() | |
61 | ||
62 | const reqVideoFileImport = createReqFiles( | |
63 | [ 'thumbnailfile', 'previewfile', 'torrentfile' ], | |
64 | { ...MIMETYPES.TORRENT.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT } | |
65 | ) | |
66 | ||
67 | videoImportsRouter.post('/imports', | |
68 | authenticate, | |
69 | reqVideoFileImport, | |
70 | asyncMiddleware(videoImportAddValidator), | |
71 | asyncRetryTransactionMiddleware(addVideoImport) | |
72 | ) | |
73 | ||
74 | videoImportsRouter.post('/imports/:id/cancel', | |
75 | authenticate, | |
76 | asyncMiddleware(videoImportCancelValidator), | |
77 | asyncRetryTransactionMiddleware(cancelVideoImport) | |
78 | ) | |
79 | ||
80 | videoImportsRouter.delete('/imports/:id', | |
81 | authenticate, | |
82 | asyncMiddleware(videoImportDeleteValidator), | |
83 | asyncRetryTransactionMiddleware(deleteVideoImport) | |
84 | ) | |
85 | ||
86 | // --------------------------------------------------------------------------- | |
87 | ||
88 | export { | |
89 | videoImportsRouter | |
90 | } | |
91 | ||
92 | // --------------------------------------------------------------------------- | |
93 | ||
94 | async function deleteVideoImport (req: express.Request, res: express.Response) { | |
95 | const videoImport = res.locals.videoImport | |
96 | ||
97 | await videoImport.destroy() | |
98 | ||
99 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | |
100 | } | |
101 | ||
102 | async function cancelVideoImport (req: express.Request, res: express.Response) { | |
103 | const videoImport = res.locals.videoImport | |
104 | ||
105 | videoImport.state = VideoImportState.CANCELLED | |
106 | await videoImport.save() | |
107 | ||
108 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | |
109 | } | |
110 | ||
111 | function addVideoImport (req: express.Request, res: express.Response) { | |
112 | if (req.body.targetUrl) return addYoutubeDLImport(req, res) | |
113 | ||
114 | const file = req.files?.['torrentfile']?.[0] | |
115 | if (req.body.magnetUri || file) return addTorrentImport(req, res, file) | |
116 | } | |
117 | ||
118 | async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { | |
119 | const body: VideoImportCreate = req.body | |
120 | const user = res.locals.oauth.token.User | |
121 | ||
122 | let videoName: string | |
123 | let torrentName: string | |
124 | let magnetUri: string | |
125 | ||
126 | if (torrentfile) { | |
127 | const result = await processTorrentOrAbortRequest(req, res, torrentfile) | |
128 | if (!result) return | |
129 | ||
130 | videoName = result.name | |
131 | torrentName = result.torrentName | |
132 | } else { | |
133 | const result = processMagnetURI(body) | |
134 | magnetUri = result.magnetUri | |
135 | videoName = result.name | |
136 | } | |
137 | ||
138 | const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName }) | |
139 | ||
140 | const thumbnailModel = await processThumbnail(req, video) | |
141 | const previewModel = await processPreview(req, video) | |
142 | ||
143 | const videoImport = await insertIntoDB({ | |
144 | video, | |
145 | thumbnailModel, | |
146 | previewModel, | |
147 | videoChannel: res.locals.videoChannel, | |
148 | tags: body.tags || undefined, | |
149 | user, | |
150 | videoImportAttributes: { | |
151 | magnetUri, | |
152 | torrentName, | |
153 | state: VideoImportState.PENDING, | |
154 | userId: user.id | |
155 | } | |
156 | }) | |
157 | ||
158 | // Create job to import the video | |
159 | const payload = { | |
160 | type: torrentfile | |
161 | ? 'torrent-file' as 'torrent-file' | |
162 | : 'magnet-uri' as 'magnet-uri', | |
163 | videoImportId: videoImport.id, | |
164 | magnetUri | |
165 | } | |
166 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | |
167 | ||
168 | auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) | |
169 | ||
170 | return res.json(videoImport.toFormattedJSON()).end() | |
171 | } | |
172 | ||
173 | async function addYoutubeDLImport (req: express.Request, res: express.Response) { | |
174 | const body: VideoImportCreate = req.body | |
175 | const targetUrl = body.targetUrl | |
176 | const user = res.locals.oauth.token.User | |
177 | ||
178 | const youtubeDL = new YoutubeDLWrapper( | |
179 | targetUrl, | |
180 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | |
181 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | |
182 | ) | |
183 | ||
184 | // Get video infos | |
185 | let youtubeDLInfo: YoutubeDLInfo | |
186 | try { | |
187 | youtubeDLInfo = await youtubeDL.getInfoForDownload() | |
188 | } catch (err) { | |
189 | logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) | |
190 | ||
191 | return res.fail({ | |
192 | message: 'Cannot fetch remote information of this URL.', | |
193 | data: { | |
194 | targetUrl | |
195 | } | |
196 | }) | |
197 | } | |
198 | ||
199 | if (!await hasUnicastURLsOnly(youtubeDLInfo)) { | |
200 | return res.fail({ | |
201 | status: HttpStatusCode.FORBIDDEN_403, | |
202 | message: 'Cannot use non unicast IP as targetUrl.' | |
203 | }) | |
204 | } | |
205 | ||
206 | const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) | |
207 | ||
208 | // Process video thumbnail from request.files | |
209 | let thumbnailModel = await processThumbnail(req, video) | |
210 | ||
211 | // Process video thumbnail from url if processing from request.files failed | |
212 | if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) { | |
213 | try { | |
214 | thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video) | |
215 | } catch (err) { | |
216 | logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err }) | |
217 | } | |
218 | } | |
219 | ||
220 | // Process video preview from request.files | |
221 | let previewModel = await processPreview(req, video) | |
222 | ||
223 | // Process video preview from url if processing from request.files failed | |
224 | if (!previewModel && youtubeDLInfo.thumbnailUrl) { | |
225 | try { | |
226 | previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video) | |
227 | } catch (err) { | |
228 | logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err }) | |
229 | } | |
230 | } | |
231 | ||
232 | const videoImport = await insertIntoDB({ | |
233 | video, | |
234 | thumbnailModel, | |
235 | previewModel, | |
236 | videoChannel: res.locals.videoChannel, | |
237 | tags: body.tags || youtubeDLInfo.tags, | |
238 | user, | |
239 | videoImportAttributes: { | |
240 | targetUrl, | |
241 | state: VideoImportState.PENDING, | |
242 | userId: user.id | |
243 | } | |
244 | }) | |
245 | ||
246 | // Get video subtitles | |
247 | await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) | |
248 | ||
249 | let fileExt = `.${youtubeDLInfo.ext}` | |
250 | if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' | |
251 | ||
252 | // Create job to import the video | |
253 | const payload = { | |
254 | type: 'youtube-dl' as 'youtube-dl', | |
255 | videoImportId: videoImport.id, | |
256 | fileExt | |
257 | } | |
258 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | |
259 | ||
260 | auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) | |
261 | ||
262 | return res.json(videoImport.toFormattedJSON()).end() | |
263 | } | |
264 | ||
265 | async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> { | |
266 | let videoData = { | |
267 | name: body.name || importData.name || 'Unknown name', | |
268 | remote: false, | |
269 | category: body.category || importData.category, | |
270 | licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, | |
271 | language: body.language || importData.language, | |
272 | commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, | |
273 | downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, | |
274 | waitTranscoding: body.waitTranscoding || false, | |
275 | state: VideoState.TO_IMPORT, | |
276 | nsfw: body.nsfw || importData.nsfw || false, | |
277 | description: body.description || importData.description, | |
278 | support: body.support || null, | |
279 | privacy: body.privacy || VideoPrivacy.PRIVATE, | |
280 | duration: 0, // duration will be set by the import job | |
281 | channelId, | |
282 | originallyPublishedAt: body.originallyPublishedAt | |
283 | ? new Date(body.originallyPublishedAt) | |
284 | : importData.originallyPublishedAt | |
285 | } | |
286 | ||
287 | videoData = await Hooks.wrapObject( | |
288 | videoData, | |
289 | body.targetUrl | |
290 | ? 'filter:api.video.import-url.video-attribute.result' | |
291 | : 'filter:api.video.import-torrent.video-attribute.result' | |
292 | ) | |
293 | ||
294 | const video = new VideoModel(videoData) | |
295 | video.url = getLocalVideoActivityPubUrl(video) | |
296 | ||
297 | return video | |
298 | } | |
299 | ||
300 | async function processThumbnail (req: express.Request, video: MVideoThumbnail) { | |
301 | const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined | |
302 | if (thumbnailField) { | |
303 | const thumbnailPhysicalFile = thumbnailField[0] | |
304 | ||
305 | return updateVideoMiniatureFromExisting({ | |
306 | inputPath: thumbnailPhysicalFile.path, | |
307 | video, | |
308 | type: ThumbnailType.MINIATURE, | |
309 | automaticallyGenerated: false | |
310 | }) | |
311 | } | |
312 | ||
313 | return undefined | |
314 | } | |
315 | ||
316 | async function processPreview (req: express.Request, video: MVideoThumbnail): Promise<MThumbnail> { | |
317 | const previewField = req.files ? req.files['previewfile'] : undefined | |
318 | if (previewField) { | |
319 | const previewPhysicalFile = previewField[0] | |
320 | ||
321 | return updateVideoMiniatureFromExisting({ | |
322 | inputPath: previewPhysicalFile.path, | |
323 | video, | |
324 | type: ThumbnailType.PREVIEW, | |
325 | automaticallyGenerated: false | |
326 | }) | |
327 | } | |
328 | ||
329 | return undefined | |
330 | } | |
331 | ||
332 | async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) { | |
333 | try { | |
334 | return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE }) | |
335 | } catch (err) { | |
336 | logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err }) | |
337 | return undefined | |
338 | } | |
339 | } | |
340 | ||
341 | async function processPreviewFromUrl (url: string, video: MVideoThumbnail) { | |
342 | try { | |
343 | return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW }) | |
344 | } catch (err) { | |
345 | logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err }) | |
346 | return undefined | |
347 | } | |
348 | } | |
349 | ||
350 | async function insertIntoDB (parameters: { | |
351 | video: MVideoThumbnail | |
352 | thumbnailModel: MThumbnail | |
353 | previewModel: MThumbnail | |
354 | videoChannel: MChannelAccountDefault | |
355 | tags: string[] | |
356 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> | |
357 | user: MUser | |
358 | }): Promise<MVideoImportFormattable> { | |
359 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters | |
360 | ||
361 | const videoImport = await sequelizeTypescript.transaction(async t => { | |
362 | const sequelizeOptions = { transaction: t } | |
363 | ||
364 | // Save video object in database | |
365 | const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) | |
366 | videoCreated.VideoChannel = videoChannel | |
367 | ||
368 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | |
369 | if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) | |
370 | ||
371 | await autoBlacklistVideoIfNeeded({ | |
372 | video: videoCreated, | |
373 | user, | |
374 | notify: false, | |
375 | isRemote: false, | |
376 | isNew: true, | |
377 | transaction: t | |
378 | }) | |
379 | ||
380 | await setVideoTags({ video: videoCreated, tags, transaction: t }) | |
381 | ||
382 | // Create video import object in database | |
383 | const videoImport = await VideoImportModel.create( | |
384 | Object.assign({ videoId: videoCreated.id }, videoImportAttributes), | |
385 | sequelizeOptions | |
386 | ) as MVideoImportFormattable | |
387 | videoImport.Video = videoCreated | |
388 | ||
389 | return videoImport | |
390 | }) | |
391 | ||
392 | return videoImport | |
393 | } | |
394 | ||
395 | async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { | |
396 | const torrentName = torrentfile.originalname | |
397 | ||
398 | // Rename the torrent to a secured name | |
399 | const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName)) | |
400 | await move(torrentfile.path, newTorrentPath, { overwrite: true }) | |
401 | torrentfile.path = newTorrentPath | |
402 | ||
403 | const buf = await readFile(torrentfile.path) | |
404 | const parsedTorrent = parseTorrent(buf) as Instance | |
405 | ||
406 | if (parsedTorrent.files.length !== 1) { | |
407 | cleanUpReqFiles(req) | |
408 | ||
409 | res.fail({ | |
410 | type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT, | |
411 | message: 'Torrents with only 1 file are supported.' | |
412 | }) | |
413 | return undefined | |
414 | } | |
415 | ||
416 | return { | |
417 | name: extractNameFromArray(parsedTorrent.name), | |
418 | torrentName | |
419 | } | |
420 | } | |
421 | ||
422 | function processMagnetURI (body: VideoImportCreate) { | |
423 | const magnetUri = body.magnetUri | |
424 | const parsed = decode(magnetUri) | |
425 | ||
426 | return { | |
427 | name: extractNameFromArray(parsed.name), | |
428 | magnetUri | |
429 | } | |
430 | } | |
431 | ||
432 | function extractNameFromArray (name: string | string[]) { | |
433 | return isArray(name) ? name[0] : name | |
434 | } | |
435 | ||
436 | async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { | |
437 | try { | |
438 | const subtitles = await youtubeDL.getSubtitles() | |
439 | ||
440 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) | |
441 | ||
442 | for (const subtitle of subtitles) { | |
443 | if (!await isVTTFileValid(subtitle.path)) { | |
444 | await remove(subtitle.path) | |
445 | continue | |
446 | } | |
447 | ||
448 | const videoCaption = new VideoCaptionModel({ | |
449 | videoId, | |
450 | language: subtitle.language, | |
451 | filename: VideoCaptionModel.generateCaptionName(subtitle.language) | |
452 | }) as MVideoCaption | |
453 | ||
454 | // Move physical file | |
455 | await moveAndProcessCaptionFile(subtitle, videoCaption) | |
456 | ||
457 | await sequelizeTypescript.transaction(async t => { | |
458 | await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) | |
459 | }) | |
460 | } | |
461 | } catch (err) { | |
462 | logger.warn('Cannot get video subtitles.', { err }) | |
463 | } | |
464 | } | |
465 | ||
466 | async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { | |
467 | const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) | |
468 | const uniqHosts = new Set(hosts) | |
469 | ||
470 | for (const h of uniqHosts) { | |
471 | if (await isResolvingToUnicastOnly(h) !== true) { | |
472 | return false | |
473 | } | |
474 | } | |
475 | ||
476 | return true | |
477 | } |