diff options
Diffstat (limited to 'server/controllers/api/videos/import.ts')
-rw-r--r-- | server/controllers/api/videos/import.ts | 170 |
1 files changed, 107 insertions, 63 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 3b9b887e2..0d5d7a962 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -3,7 +3,9 @@ import { move, readFile } from 'fs-extra' | |||
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import * as parseTorrent from 'parse-torrent' | 4 | import * as parseTorrent from 'parse-torrent' |
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { ServerConfigManager } from '@server/lib/server-config-manager' | ||
6 | import { setVideoTags } from '@server/lib/video' | 7 | import { setVideoTags } from '@server/lib/video' |
8 | import { FilteredModelAttributes } from '@server/types' | ||
7 | import { | 9 | import { |
8 | MChannelAccountDefault, | 10 | MChannelAccountDefault, |
9 | MThumbnail, | 11 | MThumbnail, |
@@ -14,17 +16,17 @@ import { | |||
14 | MVideoThumbnail, | 16 | MVideoThumbnail, |
15 | MVideoWithBlacklistLight | 17 | MVideoWithBlacklistLight |
16 | } from '@server/types/models' | 18 | } from '@server/types/models' |
17 | import { MVideoImport, MVideoImportFormattable } from '@server/types/models/video/video-import' | 19 | import { MVideoImportFormattable } from '@server/types/models/video/video-import' |
18 | import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' | 20 | import { ServerErrorCode, VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' |
19 | import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' | 21 | import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' |
20 | import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' | 22 | import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' |
21 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' | 23 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' |
22 | import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' | 24 | import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' |
23 | import { isArray } from '../../../helpers/custom-validators/misc' | 25 | import { isArray } from '../../../helpers/custom-validators/misc' |
24 | import { createReqFiles } from '../../../helpers/express-utils' | 26 | import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' |
25 | import { logger } from '../../../helpers/logger' | 27 | import { logger } from '../../../helpers/logger' |
26 | import { getSecureTorrentName } from '../../../helpers/utils' | 28 | import { getSecureTorrentName } from '../../../helpers/utils' |
27 | import { getYoutubeDLInfo, getYoutubeDLSubs, YoutubeDLInfo } from '../../../helpers/youtube-dl' | 29 | import { YoutubeDL, YoutubeDLInfo } from '../../../helpers/youtube-dl' |
28 | import { CONFIG } from '../../../initializers/config' | 30 | import { CONFIG } from '../../../initializers/config' |
29 | import { MIMETYPES } from '../../../initializers/constants' | 31 | import { MIMETYPES } from '../../../initializers/constants' |
30 | import { sequelizeTypescript } from '../../../initializers/database' | 32 | import { sequelizeTypescript } from '../../../initializers/database' |
@@ -81,22 +83,15 @@ async function addTorrentImport (req: express.Request, res: express.Response, to | |||
81 | let magnetUri: string | 83 | let magnetUri: string |
82 | 84 | ||
83 | if (torrentfile) { | 85 | if (torrentfile) { |
84 | torrentName = torrentfile.originalname | 86 | const result = await processTorrentOrAbortRequest(req, res, torrentfile) |
87 | if (!result) return | ||
85 | 88 | ||
86 | // Rename the torrent to a secured name | 89 | videoName = result.name |
87 | const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName)) | 90 | torrentName = result.torrentName |
88 | await move(torrentfile.path, newTorrentPath) | ||
89 | torrentfile.path = newTorrentPath | ||
90 | |||
91 | const buf = await readFile(torrentfile.path) | ||
92 | const parsedTorrent = parseTorrent(buf) | ||
93 | |||
94 | videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[0] : parsedTorrent.name as string | ||
95 | } else { | 91 | } else { |
96 | magnetUri = body.magnetUri | 92 | const result = processMagnetURI(body) |
97 | 93 | magnetUri = result.magnetUri | |
98 | const parsed = magnetUtil.decode(magnetUri) | 94 | videoName = result.name |
99 | videoName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string | ||
100 | } | 95 | } |
101 | 96 | ||
102 | const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName }) | 97 | const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName }) |
@@ -104,26 +99,26 @@ async function addTorrentImport (req: express.Request, res: express.Response, to | |||
104 | const thumbnailModel = await processThumbnail(req, video) | 99 | const thumbnailModel = await processThumbnail(req, video) |
105 | const previewModel = await processPreview(req, video) | 100 | const previewModel = await processPreview(req, video) |
106 | 101 | ||
107 | const tags = body.tags || undefined | ||
108 | const videoImportAttributes = { | ||
109 | magnetUri, | ||
110 | torrentName, | ||
111 | state: VideoImportState.PENDING, | ||
112 | userId: user.id | ||
113 | } | ||
114 | const videoImport = await insertIntoDB({ | 102 | const videoImport = await insertIntoDB({ |
115 | video, | 103 | video, |
116 | thumbnailModel, | 104 | thumbnailModel, |
117 | previewModel, | 105 | previewModel, |
118 | videoChannel: res.locals.videoChannel, | 106 | videoChannel: res.locals.videoChannel, |
119 | tags, | 107 | tags: body.tags || undefined, |
120 | videoImportAttributes, | 108 | user, |
121 | user | 109 | videoImportAttributes: { |
110 | magnetUri, | ||
111 | torrentName, | ||
112 | state: VideoImportState.PENDING, | ||
113 | userId: user.id | ||
114 | } | ||
122 | }) | 115 | }) |
123 | 116 | ||
124 | // Create job to import the video | 117 | // Create job to import the video |
125 | const payload = { | 118 | const payload = { |
126 | type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri', | 119 | type: torrentfile |
120 | ? 'torrent-file' as 'torrent-file' | ||
121 | : 'magnet-uri' as 'magnet-uri', | ||
127 | videoImportId: videoImport.id, | 122 | videoImportId: videoImport.id, |
128 | magnetUri | 123 | magnetUri |
129 | } | 124 | } |
@@ -139,10 +134,12 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) | |||
139 | const targetUrl = body.targetUrl | 134 | const targetUrl = body.targetUrl |
140 | const user = res.locals.oauth.token.User | 135 | const user = res.locals.oauth.token.User |
141 | 136 | ||
137 | const youtubeDL = new YoutubeDL(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod')) | ||
138 | |||
142 | // Get video infos | 139 | // Get video infos |
143 | let youtubeDLInfo: YoutubeDLInfo | 140 | let youtubeDLInfo: YoutubeDLInfo |
144 | try { | 141 | try { |
145 | youtubeDLInfo = await getYoutubeDLInfo(targetUrl) | 142 | youtubeDLInfo = await youtubeDL.getYoutubeDLInfo() |
146 | } catch (err) { | 143 | } catch (err) { |
147 | logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) | 144 | logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) |
148 | 145 | ||
@@ -170,45 +167,22 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) | |||
170 | previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video) | 167 | previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video) |
171 | } | 168 | } |
172 | 169 | ||
173 | const tags = body.tags || youtubeDLInfo.tags | ||
174 | const videoImportAttributes = { | ||
175 | targetUrl, | ||
176 | state: VideoImportState.PENDING, | ||
177 | userId: user.id | ||
178 | } | ||
179 | const videoImport = await insertIntoDB({ | 170 | const videoImport = await insertIntoDB({ |
180 | video, | 171 | video, |
181 | thumbnailModel, | 172 | thumbnailModel, |
182 | previewModel, | 173 | previewModel, |
183 | videoChannel: res.locals.videoChannel, | 174 | videoChannel: res.locals.videoChannel, |
184 | tags, | 175 | tags: body.tags || youtubeDLInfo.tags, |
185 | videoImportAttributes, | 176 | user, |
186 | user | 177 | videoImportAttributes: { |
178 | targetUrl, | ||
179 | state: VideoImportState.PENDING, | ||
180 | userId: user.id | ||
181 | } | ||
187 | }) | 182 | }) |
188 | 183 | ||
189 | // Get video subtitles | 184 | // Get video subtitles |
190 | try { | 185 | await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) |
191 | const subtitles = await getYoutubeDLSubs(targetUrl) | ||
192 | |||
193 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) | ||
194 | |||
195 | for (const subtitle of subtitles) { | ||
196 | const videoCaption = new VideoCaptionModel({ | ||
197 | videoId: video.id, | ||
198 | language: subtitle.language, | ||
199 | filename: VideoCaptionModel.generateCaptionName(subtitle.language) | ||
200 | }) as MVideoCaption | ||
201 | |||
202 | // Move physical file | ||
203 | await moveAndProcessCaptionFile(subtitle, videoCaption) | ||
204 | |||
205 | await sequelizeTypescript.transaction(async t => { | ||
206 | await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) | ||
207 | }) | ||
208 | } | ||
209 | } catch (err) { | ||
210 | logger.warn('Cannot get video subtitles.', { err }) | ||
211 | } | ||
212 | 186 | ||
213 | // Create job to import the video | 187 | // Create job to import the video |
214 | const payload = { | 188 | const payload = { |
@@ -240,7 +214,9 @@ function buildVideo (channelId: number, body: VideoImportCreate, importData: You | |||
240 | privacy: body.privacy || VideoPrivacy.PRIVATE, | 214 | privacy: body.privacy || VideoPrivacy.PRIVATE, |
241 | duration: 0, // duration will be set by the import job | 215 | duration: 0, // duration will be set by the import job |
242 | channelId: channelId, | 216 | channelId: channelId, |
243 | originallyPublishedAt: body.originallyPublishedAt || importData.originallyPublishedAt | 217 | originallyPublishedAt: body.originallyPublishedAt |
218 | ? new Date(body.originallyPublishedAt) | ||
219 | : importData.originallyPublishedAt | ||
244 | } | 220 | } |
245 | const video = new VideoModel(videoData) | 221 | const video = new VideoModel(videoData) |
246 | video.url = getLocalVideoActivityPubUrl(video) | 222 | video.url = getLocalVideoActivityPubUrl(video) |
@@ -304,7 +280,7 @@ async function insertIntoDB (parameters: { | |||
304 | previewModel: MThumbnail | 280 | previewModel: MThumbnail |
305 | videoChannel: MChannelAccountDefault | 281 | videoChannel: MChannelAccountDefault |
306 | tags: string[] | 282 | tags: string[] |
307 | videoImportAttributes: Partial<MVideoImport> | 283 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> |
308 | user: MUser | 284 | user: MUser |
309 | }): Promise<MVideoImportFormattable> { | 285 | }): Promise<MVideoImportFormattable> { |
310 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters | 286 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters |
@@ -342,3 +318,71 @@ async function insertIntoDB (parameters: { | |||
342 | 318 | ||
343 | return videoImport | 319 | return videoImport |
344 | } | 320 | } |
321 | |||
322 | async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { | ||
323 | const torrentName = torrentfile.originalname | ||
324 | |||
325 | // Rename the torrent to a secured name | ||
326 | const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName)) | ||
327 | await move(torrentfile.path, newTorrentPath, { overwrite: true }) | ||
328 | torrentfile.path = newTorrentPath | ||
329 | |||
330 | const buf = await readFile(torrentfile.path) | ||
331 | const parsedTorrent = parseTorrent(buf) as parseTorrent.Instance | ||
332 | |||
333 | if (parsedTorrent.files.length !== 1) { | ||
334 | cleanUpReqFiles(req) | ||
335 | |||
336 | res.status(HttpStatusCode.BAD_REQUEST_400) | ||
337 | .json({ | ||
338 | code: ServerErrorCode.INCORRECT_FILES_IN_TORRENT, | ||
339 | error: 'Torrents with only 1 file are supported.' | ||
340 | }) | ||
341 | |||
342 | return undefined | ||
343 | } | ||
344 | |||
345 | return { | ||
346 | name: extractNameFromArray(parsedTorrent.name), | ||
347 | torrentName | ||
348 | } | ||
349 | } | ||
350 | |||
351 | function processMagnetURI (body: VideoImportCreate) { | ||
352 | const magnetUri = body.magnetUri | ||
353 | const parsed = magnetUtil.decode(magnetUri) | ||
354 | |||
355 | return { | ||
356 | name: extractNameFromArray(parsed.name), | ||
357 | magnetUri | ||
358 | } | ||
359 | } | ||
360 | |||
361 | function extractNameFromArray (name: string | string[]) { | ||
362 | return isArray(name) ? name[0] : name | ||
363 | } | ||
364 | |||
365 | async function processYoutubeSubtitles (youtubeDL: YoutubeDL, targetUrl: string, videoId: number) { | ||
366 | try { | ||
367 | const subtitles = await youtubeDL.getYoutubeDLSubs() | ||
368 | |||
369 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) | ||
370 | |||
371 | for (const subtitle of subtitles) { | ||
372 | const videoCaption = new VideoCaptionModel({ | ||
373 | videoId, | ||
374 | language: subtitle.language, | ||
375 | filename: VideoCaptionModel.generateCaptionName(subtitle.language) | ||
376 | }) as MVideoCaption | ||
377 | |||
378 | // Move physical file | ||
379 | await moveAndProcessCaptionFile(subtitle, videoCaption) | ||
380 | |||
381 | await sequelizeTypescript.transaction(async t => { | ||
382 | await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) | ||
383 | }) | ||
384 | } | ||
385 | } catch (err) { | ||
386 | logger.warn('Cannot get video subtitles.', { err }) | ||
387 | } | ||
388 | } | ||