]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blob - server/controllers/api/videos/import.ts
Split types and typings
[github/Chocobozzz/PeerTube.git] / server / controllers / api / videos / import.ts
1 import * as express from 'express'
2 import * as magnetUtil from 'magnet-uri'
3 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
4 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
5 import { MIMETYPES } from '../../../initializers/constants'
6 import { getYoutubeDLInfo, YoutubeDLInfo, getYoutubeDLSubs } from '../../../helpers/youtube-dl'
7 import { createReqFiles } from '../../../helpers/express-utils'
8 import { logger } from '../../../helpers/logger'
9 import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
10 import { VideoModel } from '../../../models/video/video'
11 import { VideoCaptionModel } from '../../../models/video/video-caption'
12 import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
13 import { getVideoActivityPubUrl } from '../../../lib/activitypub/url'
14 import { TagModel } from '../../../models/video/tag'
15 import { VideoImportModel } from '../../../models/video/video-import'
16 import { JobQueue } from '../../../lib/job-queue/job-queue'
17 import { join } from 'path'
18 import { isArray } from '../../../helpers/custom-validators/misc'
19 import * as Bluebird from 'bluebird'
20 import * as parseTorrent from 'parse-torrent'
21 import { getSecureTorrentName } from '../../../helpers/utils'
22 import { move, readFile } from 'fs-extra'
23 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
24 import { CONFIG } from '../../../initializers/config'
25 import { sequelizeTypescript } from '../../../initializers/database'
26 import { createVideoMiniatureFromExisting, createVideoMiniatureFromUrl } from '../../../lib/thumbnail'
27 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
28 import {
29 MChannelAccountDefault,
30 MThumbnail,
31 MUser,
32 MVideoAccountDefault,
33 MVideoCaptionVideo,
34 MVideoTag,
35 MVideoThumbnailAccountDefault,
36 MVideoWithBlacklistLight
37 } from '@server/types/models'
38 import { MVideoImport, MVideoImportFormattable } from '@server/types/models/video/video-import'
39
40 const auditLogger = auditLoggerFactory('video-imports')
41 const videoImportsRouter = express.Router()
42
43 const reqVideoFileImport = createReqFiles(
44 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
45 Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
46 {
47 thumbnailfile: CONFIG.STORAGE.TMP_DIR,
48 previewfile: CONFIG.STORAGE.TMP_DIR,
49 torrentfile: CONFIG.STORAGE.TMP_DIR
50 }
51 )
52
53 videoImportsRouter.post('/imports',
54 authenticate,
55 reqVideoFileImport,
56 asyncMiddleware(videoImportAddValidator),
57 asyncRetryTransactionMiddleware(addVideoImport)
58 )
59
60 // ---------------------------------------------------------------------------
61
62 export {
63 videoImportsRouter
64 }
65
66 // ---------------------------------------------------------------------------
67
68 function addVideoImport (req: express.Request, res: express.Response) {
69 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
70
71 const file = req.files?.['torrentfile']?.[0]
72 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
73 }
74
75 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
76 const body: VideoImportCreate = req.body
77 const user = res.locals.oauth.token.User
78
79 let videoName: string
80 let torrentName: string
81 let magnetUri: string
82
83 if (torrentfile) {
84 torrentName = torrentfile.originalname
85
86 // Rename the torrent to a secured name
87 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
88 await move(torrentfile.path, newTorrentPath)
89 torrentfile.path = newTorrentPath
90
91 const buf = await readFile(torrentfile.path)
92 const parsedTorrent = parseTorrent(buf)
93
94 videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[0] : parsedTorrent.name as string
95 } else {
96 magnetUri = body.magnetUri
97
98 const parsed = magnetUtil.decode(magnetUri)
99 videoName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string
100 }
101
102 const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName })
103
104 const thumbnailModel = await processThumbnail(req, video)
105 const previewModel = await processPreview(req, video)
106
107 const tags = body.tags || undefined
108 const videoImportAttributes = {
109 magnetUri,
110 torrentName,
111 state: VideoImportState.PENDING,
112 userId: user.id
113 }
114 const videoImport = await insertIntoDB({
115 video,
116 thumbnailModel,
117 previewModel,
118 videoChannel: res.locals.videoChannel,
119 tags,
120 videoImportAttributes,
121 user
122 })
123
124 // Create job to import the video
125 const payload = {
126 type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri',
127 videoImportId: videoImport.id,
128 magnetUri
129 }
130 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
131
132 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
133
134 return res.json(videoImport.toFormattedJSON()).end()
135 }
136
137 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
138 const body: VideoImportCreate = req.body
139 const targetUrl = body.targetUrl
140 const user = res.locals.oauth.token.User
141
142 // Get video infos
143 let youtubeDLInfo: YoutubeDLInfo
144 try {
145 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
146 } catch (err) {
147 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
148
149 return res.status(400).json({
150 error: 'Cannot fetch remote information of this URL.'
151 }).end()
152 }
153
154 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
155
156 let thumbnailModel: MThumbnail
157
158 // Process video thumbnail from request.files
159 thumbnailModel = await processThumbnail(req, video)
160
161 // Process video thumbnail from url if processing from request.files failed
162 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
163 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
164 }
165
166 let previewModel: MThumbnail
167
168 // Process video preview from request.files
169 previewModel = await processPreview(req, video)
170
171 // Process video preview from url if processing from request.files failed
172 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
173 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
174 }
175
176 const tags = body.tags || youtubeDLInfo.tags
177 const videoImportAttributes = {
178 targetUrl,
179 state: VideoImportState.PENDING,
180 userId: user.id
181 }
182 const videoImport = await insertIntoDB({
183 video,
184 thumbnailModel,
185 previewModel,
186 videoChannel: res.locals.videoChannel,
187 tags,
188 videoImportAttributes,
189 user
190 })
191
192 // Get video subtitles
193 try {
194 const subtitles = await getYoutubeDLSubs(targetUrl)
195
196 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
197
198 for (const subtitle of subtitles) {
199 const videoCaption = new VideoCaptionModel({
200 videoId: video.id,
201 language: subtitle.language
202 }) as MVideoCaptionVideo
203 videoCaption.Video = video
204
205 // Move physical file
206 await moveAndProcessCaptionFile(subtitle, videoCaption)
207
208 await sequelizeTypescript.transaction(async t => {
209 await VideoCaptionModel.insertOrReplaceLanguage(video.id, subtitle.language, null, t)
210 })
211 }
212 } catch (err) {
213 logger.warn('Cannot get video subtitles.', { err })
214 }
215
216 // Create job to import the video
217 const payload = {
218 type: 'youtube-dl' as 'youtube-dl',
219 videoImportId: videoImport.id,
220 generateThumbnail: !thumbnailModel,
221 generatePreview: !previewModel,
222 fileExt: youtubeDLInfo.fileExt
223 ? `.${youtubeDLInfo.fileExt}`
224 : '.mp4'
225 }
226 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
227
228 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
229
230 return res.json(videoImport.toFormattedJSON()).end()
231 }
232
233 function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
234 const videoData = {
235 name: body.name || importData.name || 'Unknown name',
236 remote: false,
237 category: body.category || importData.category,
238 licence: body.licence || importData.licence,
239 language: body.language || importData.language,
240 commentsEnabled: body.commentsEnabled !== false, // If the value is not "false", the default is "true"
241 downloadEnabled: body.downloadEnabled !== false,
242 waitTranscoding: body.waitTranscoding || false,
243 state: VideoState.TO_IMPORT,
244 nsfw: body.nsfw || importData.nsfw || false,
245 description: body.description || importData.description,
246 support: body.support || null,
247 privacy: body.privacy || VideoPrivacy.PRIVATE,
248 duration: 0, // duration will be set by the import job
249 channelId: channelId,
250 originallyPublishedAt: body.originallyPublishedAt || importData.originallyPublishedAt
251 }
252 const video = new VideoModel(videoData)
253 video.url = getVideoActivityPubUrl(video)
254
255 return video
256 }
257
258 async function processThumbnail (req: express.Request, video: VideoModel) {
259 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
260 if (thumbnailField) {
261 const thumbnailPhysicalFile = thumbnailField[0]
262
263 return createVideoMiniatureFromExisting(thumbnailPhysicalFile.path, video, ThumbnailType.MINIATURE, false)
264 }
265
266 return undefined
267 }
268
269 async function processPreview (req: express.Request, video: VideoModel) {
270 const previewField = req.files ? req.files['previewfile'] : undefined
271 if (previewField) {
272 const previewPhysicalFile = previewField[0]
273
274 return createVideoMiniatureFromExisting(previewPhysicalFile.path, video, ThumbnailType.PREVIEW, false)
275 }
276
277 return undefined
278 }
279
280 async function processThumbnailFromUrl (url: string, video: VideoModel) {
281 try {
282 return createVideoMiniatureFromUrl(url, video, ThumbnailType.MINIATURE)
283 } catch (err) {
284 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
285 return undefined
286 }
287 }
288
289 async function processPreviewFromUrl (url: string, video: VideoModel) {
290 try {
291 return createVideoMiniatureFromUrl(url, video, ThumbnailType.PREVIEW)
292 } catch (err) {
293 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
294 return undefined
295 }
296 }
297
298 function insertIntoDB (parameters: {
299 video: MVideoThumbnailAccountDefault
300 thumbnailModel: MThumbnail
301 previewModel: MThumbnail
302 videoChannel: MChannelAccountDefault
303 tags: string[]
304 videoImportAttributes: Partial<MVideoImport>
305 user: MUser
306 }): Bluebird<MVideoImportFormattable> {
307 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
308
309 return sequelizeTypescript.transaction(async t => {
310 const sequelizeOptions = { transaction: t }
311
312 // Save video object in database
313 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
314 videoCreated.VideoChannel = videoChannel
315
316 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
317 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
318
319 await autoBlacklistVideoIfNeeded({
320 video: videoCreated,
321 user,
322 notify: false,
323 isRemote: false,
324 isNew: true,
325 transaction: t
326 })
327
328 // Set tags to the video
329 if (tags) {
330 const tagInstances = await TagModel.findOrCreateTags(tags, t)
331
332 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
333 videoCreated.Tags = tagInstances
334 } else {
335 videoCreated.Tags = []
336 }
337
338 // Create video import object in database
339 const videoImport = await VideoImportModel.create(
340 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
341 sequelizeOptions
342 ) as MVideoImportFormattable
343 videoImport.Video = videoCreated
344
345 return videoImport
346 })
347 }