aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/controllers/api/videos
diff options
context:
space:
mode:
authorFlorent <florent.git@zeteo.me>2022-08-10 09:53:39 +0200
committerGitHub <noreply@github.com>2022-08-10 09:53:39 +0200
commit2a491182e483b97afb1b65c908b23cb48d591807 (patch)
treeec13503216ad72a3ea8f1ce3b659899f8167fb47 /server/controllers/api/videos
parent06ac128958c489efe1008eeca1df683819bd2f18 (diff)
downloadPeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.gz
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.zst
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.zip
Channel sync (#5135)
* Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/controllers/api/videos')
-rw-r--r--server/controllers/api/videos/import.ts318
1 files changed, 51 insertions, 267 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 5a2e1006a..9d7b0260b 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,49 +1,20 @@
1import express from 'express' 1import express from 'express'
2import { move, readFile, remove } from 'fs-extra' 2import { move, readFile } from 'fs-extra'
3import { decode } from 'magnet-uri' 3import { decode } from 'magnet-uri'
4import parseTorrent, { Instance } from 'parse-torrent' 4import parseTorrent, { Instance } from 'parse-torrent'
5import { join } from 'path' 5import { join } from 'path'
6import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' 6import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
7import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' 7import { MThumbnail, MVideoThumbnail } from '@server/types/models'
8import { isResolvingToUnicastOnly } from '@server/helpers/dns' 8import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
9import { Hooks } from '@server/lib/plugins/hooks'
10import { ServerConfigManager } from '@server/lib/server-config-manager'
11import { setVideoTags } from '@server/lib/video'
12import { FilteredModelAttributes } from '@server/types'
13import {
14 MChannelAccountDefault,
15 MThumbnail,
16 MUser,
17 MVideoAccountDefault,
18 MVideoCaption,
19 MVideoTag,
20 MVideoThumbnail,
21 MVideoWithBlacklistLight
22} from '@server/types/models'
23import { MVideoImportFormattable } from '@server/types/models/video/video-import'
24import {
25 HttpStatusCode,
26 ServerErrorCode,
27 ThumbnailType,
28 VideoImportCreate,
29 VideoImportState,
30 VideoPrivacy,
31 VideoState
32} from '@shared/models'
33import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' 9import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
34import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
35import { isArray } from '../../../helpers/custom-validators/misc' 10import { isArray } from '../../../helpers/custom-validators/misc'
36import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' 11import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
37import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
38import { getSecureTorrentName } from '../../../helpers/utils' 13import { getSecureTorrentName } from '../../../helpers/utils'
39import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
40import { CONFIG } from '../../../initializers/config' 14import { CONFIG } from '../../../initializers/config'
41import { MIMETYPES } from '../../../initializers/constants' 15import { MIMETYPES } from '../../../initializers/constants'
42import { sequelizeTypescript } from '../../../initializers/database'
43import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
44import { JobQueue } from '../../../lib/job-queue/job-queue' 16import { JobQueue } from '../../../lib/job-queue/job-queue'
45import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail' 17import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
46import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
47import { 18import {
48 asyncMiddleware, 19 asyncMiddleware,
49 asyncRetryTransactionMiddleware, 20 asyncRetryTransactionMiddleware,
@@ -52,9 +23,6 @@ import {
52 videoImportCancelValidator, 23 videoImportCancelValidator,
53 videoImportDeleteValidator 24 videoImportDeleteValidator
54} from '../../../middlewares' 25} from '../../../middlewares'
55import { VideoModel } from '../../../models/video/video'
56import { VideoCaptionModel } from '../../../models/video/video-caption'
57import { VideoImportModel } from '../../../models/video/video-import'
58 26
59const auditLogger = auditLoggerFactory('video-imports') 27const auditLogger = auditLoggerFactory('video-imports')
60const videoImportsRouter = express.Router() 28const videoImportsRouter = express.Router()
@@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
68 authenticate, 36 authenticate,
69 reqVideoFileImport, 37 reqVideoFileImport,
70 asyncMiddleware(videoImportAddValidator), 38 asyncMiddleware(videoImportAddValidator),
71 asyncRetryTransactionMiddleware(addVideoImport) 39 asyncRetryTransactionMiddleware(handleVideoImport)
72) 40)
73 41
74videoImportsRouter.post('/imports/:id/cancel', 42videoImportsRouter.post('/imports/:id/cancel',
@@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
108 return res.sendStatus(HttpStatusCode.NO_CONTENT_204) 76 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
109} 77}
110 78
111function addVideoImport (req: express.Request, res: express.Response) { 79function handleVideoImport (req: express.Request, res: express.Response) {
112 if (req.body.targetUrl) return addYoutubeDLImport(req, res) 80 if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
113 81
114 const file = req.files?.['torrentfile']?.[0] 82 const file = req.files?.['torrentfile']?.[0]
115 if (req.body.magnetUri || file) return addTorrentImport(req, res, file) 83 if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
116} 84}
117 85
118async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { 86async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
119 const body: VideoImportCreate = req.body 87 const body: VideoImportCreate = req.body
120 const user = res.locals.oauth.token.User 88 const user = res.locals.oauth.token.User
121 89
@@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
135 videoName = result.name 103 videoName = result.name
136 } 104 }
137 105
138 const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName }) 106 const video = await buildVideoFromImport({
107 channelId: res.locals.videoChannel.id,
108 importData: { name: videoName },
109 importDataOverride: body,
110 importType: 'torrent'
111 })
139 112
140 const thumbnailModel = await processThumbnail(req, video) 113 const thumbnailModel = await processThumbnail(req, video)
141 const previewModel = await processPreview(req, video) 114 const previewModel = await processPreview(req, video)
142 115
143 const videoImport = await insertIntoDB({ 116 const videoImport = await insertFromImportIntoDB({
144 video, 117 video,
145 thumbnailModel, 118 thumbnailModel,
146 previewModel, 119 previewModel,
@@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
155 } 128 }
156 }) 129 })
157 130
158 // Create job to import the video 131 const payload: VideoImportPayload = {
159 const payload = {
160 type: torrentfile 132 type: torrentfile
161 ? 'torrent-file' as 'torrent-file' 133 ? 'torrent-file'
162 : 'magnet-uri' as 'magnet-uri', 134 : 'magnet-uri',
163 videoImportId: videoImport.id, 135 videoImportId: videoImport.id,
164 magnetUri 136 preventException: false
165 } 137 }
166 await JobQueue.Instance.createJob({ type: 'video-import', payload }) 138 await JobQueue.Instance.createJob({ type: 'video-import', payload })
167 139
@@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
170 return res.json(videoImport.toFormattedJSON()).end() 142 return res.json(videoImport.toFormattedJSON()).end()
171} 143}
172 144
173async function addYoutubeDLImport (req: express.Request, res: express.Response) { 145function statusFromYtDlImportError (err: YoutubeDlImportError): number {
146 switch (err.code) {
147 case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
148 return HttpStatusCode.FORBIDDEN_403
149
150 case YoutubeDlImportError.CODE.FETCH_ERROR:
151 return HttpStatusCode.BAD_REQUEST_400
152
153 default:
154 return HttpStatusCode.INTERNAL_SERVER_ERROR_500
155 }
156}
157
158async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
174 const body: VideoImportCreate = req.body 159 const body: VideoImportCreate = req.body
175 const targetUrl = body.targetUrl 160 const targetUrl = body.targetUrl
176 const user = res.locals.oauth.token.User 161 const user = res.locals.oauth.token.User
177 162
178 const youtubeDL = new YoutubeDLWrapper(
179 targetUrl,
180 ServerConfigManager.Instance.getEnabledResolutions('vod'),
181 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
182 )
183
184 // Get video infos
185 let youtubeDLInfo: YoutubeDLInfo
186 try { 163 try {
187 youtubeDLInfo = await youtubeDL.getInfoForDownload() 164 const { job, videoImport } = await buildYoutubeDLImport({
165 targetUrl,
166 channel: res.locals.videoChannel,
167 importDataOverride: body,
168 thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
169 previewFilePath: req.files?.['previewfile']?.[0].path,
170 user
171 })
172 await JobQueue.Instance.createJob(job)
173
174 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
175
176 return res.json(videoImport.toFormattedJSON()).end()
188 } catch (err) { 177 } catch (err) {
189 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) 178 logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
190 179
191 return res.fail({ 180 return res.fail({
192 message: 'Cannot fetch remote information of this URL.', 181 message: err.message,
182 status: statusFromYtDlImportError(err),
193 data: { 183 data: {
194 targetUrl 184 targetUrl
195 } 185 }
196 }) 186 })
197 } 187 }
198
199 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
200 return res.fail({
201 status: HttpStatusCode.FORBIDDEN_403,
202 message: 'Cannot use non unicast IP as targetUrl.'
203 })
204 }
205
206 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
207
208 // Process video thumbnail from request.files
209 let thumbnailModel = await processThumbnail(req, video)
210
211 // Process video thumbnail from url if processing from request.files failed
212 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
213 try {
214 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
215 } catch (err) {
216 logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
217 }
218 }
219
220 // Process video preview from request.files
221 let previewModel = await processPreview(req, video)
222
223 // Process video preview from url if processing from request.files failed
224 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
225 try {
226 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
227 } catch (err) {
228 logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
229 }
230 }
231
232 const videoImport = await insertIntoDB({
233 video,
234 thumbnailModel,
235 previewModel,
236 videoChannel: res.locals.videoChannel,
237 tags: body.tags || youtubeDLInfo.tags,
238 user,
239 videoImportAttributes: {
240 targetUrl,
241 state: VideoImportState.PENDING,
242 userId: user.id
243 }
244 })
245
246 // Get video subtitles
247 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
248
249 let fileExt = `.${youtubeDLInfo.ext}`
250 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
251
252 // Create job to import the video
253 const payload = {
254 type: 'youtube-dl' as 'youtube-dl',
255 videoImportId: videoImport.id,
256 fileExt
257 }
258 await JobQueue.Instance.createJob({ type: 'video-import', payload })
259
260 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
261
262 return res.json(videoImport.toFormattedJSON()).end()
263}
264
265async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
266 let videoData = {
267 name: body.name || importData.name || 'Unknown name',
268 remote: false,
269 category: body.category || importData.category,
270 licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
271 language: body.language || importData.language,
272 commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
273 downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
274 waitTranscoding: body.waitTranscoding || false,
275 state: VideoState.TO_IMPORT,
276 nsfw: body.nsfw || importData.nsfw || false,
277 description: body.description || importData.description,
278 support: body.support || null,
279 privacy: body.privacy || VideoPrivacy.PRIVATE,
280 duration: 0, // duration will be set by the import job
281 channelId,
282 originallyPublishedAt: body.originallyPublishedAt
283 ? new Date(body.originallyPublishedAt)
284 : importData.originallyPublishedAt
285 }
286
287 videoData = await Hooks.wrapObject(
288 videoData,
289 body.targetUrl
290 ? 'filter:api.video.import-url.video-attribute.result'
291 : 'filter:api.video.import-torrent.video-attribute.result'
292 )
293
294 const video = new VideoModel(videoData)
295 video.url = getLocalVideoActivityPubUrl(video)
296
297 return video
298} 188}
299 189
300async function processThumbnail (req: express.Request, video: MVideoThumbnail) { 190async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
@@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
329 return undefined 219 return undefined
330} 220}
331 221
332async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
333 try {
334 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
335 } catch (err) {
336 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
337 return undefined
338 }
339}
340
341async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
342 try {
343 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
344 } catch (err) {
345 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
346 return undefined
347 }
348}
349
350async function insertIntoDB (parameters: {
351 video: MVideoThumbnail
352 thumbnailModel: MThumbnail
353 previewModel: MThumbnail
354 videoChannel: MChannelAccountDefault
355 tags: string[]
356 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
357 user: MUser
358}): Promise<MVideoImportFormattable> {
359 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
360
361 const videoImport = await sequelizeTypescript.transaction(async t => {
362 const sequelizeOptions = { transaction: t }
363
364 // Save video object in database
365 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
366 videoCreated.VideoChannel = videoChannel
367
368 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
369 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
370
371 await autoBlacklistVideoIfNeeded({
372 video: videoCreated,
373 user,
374 notify: false,
375 isRemote: false,
376 isNew: true,
377 transaction: t
378 })
379
380 await setVideoTags({ video: videoCreated, tags, transaction: t })
381
382 // Create video import object in database
383 const videoImport = await VideoImportModel.create(
384 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
385 sequelizeOptions
386 ) as MVideoImportFormattable
387 videoImport.Video = videoCreated
388
389 return videoImport
390 })
391
392 return videoImport
393}
394
395async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { 222async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
396 const torrentName = torrentfile.originalname 223 const torrentName = torrentfile.originalname
397 224
@@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
432function extractNameFromArray (name: string | string[]) { 259function extractNameFromArray (name: string | string[]) {
433 return isArray(name) ? name[0] : name 260 return isArray(name) ? name[0] : name
434} 261}
435
436async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
437 try {
438 const subtitles = await youtubeDL.getSubtitles()
439
440 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
441
442 for (const subtitle of subtitles) {
443 if (!await isVTTFileValid(subtitle.path)) {
444 await remove(subtitle.path)
445 continue
446 }
447
448 const videoCaption = new VideoCaptionModel({
449 videoId,
450 language: subtitle.language,
451 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
452 }) as MVideoCaption
453
454 // Move physical file
455 await moveAndProcessCaptionFile(subtitle, videoCaption)
456
457 await sequelizeTypescript.transaction(async t => {
458 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
459 })
460 }
461 } catch (err) {
462 logger.warn('Cannot get video subtitles.', { err })
463 }
464}
465
466async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
467 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
468 const uniqHosts = new Set(hosts)
469
470 for (const h of uniqHosts) {
471 if (await isResolvingToUnicastOnly(h) !== true) {
472 return false
473 }
474 }
475
476 return true
477}