aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/controllers/api
diff options
context:
space:
mode:
authorFlorent <florent.git@zeteo.me>2022-08-10 09:53:39 +0200
committerGitHub <noreply@github.com>2022-08-10 09:53:39 +0200
commit2a491182e483b97afb1b65c908b23cb48d591807 (patch)
treeec13503216ad72a3ea8f1ce3b659899f8167fb47 /server/controllers/api
parent06ac128958c489efe1008eeca1df683819bd2f18 (diff)
downloadPeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.gz
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.zst
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.zip
Channel sync (#5135)
* Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/controllers/api')
-rw-r--r--server/controllers/api/accounts.ts28
-rw-r--r--server/controllers/api/config.ts4
-rw-r--r--server/controllers/api/index.ts2
-rw-r--r--server/controllers/api/server/debug.ts4
-rw-r--r--server/controllers/api/video-channel-sync.ts76
-rw-r--r--server/controllers/api/video-channel.ts28
-rw-r--r--server/controllers/api/videos/import.ts318
7 files changed, 192 insertions, 268 deletions
diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts
index 66cdaab82..7a530cde5 100644
--- a/server/controllers/api/accounts.ts
+++ b/server/controllers/api/accounts.ts
@@ -25,8 +25,10 @@ import {
25 accountsFollowersSortValidator, 25 accountsFollowersSortValidator,
26 accountsSortValidator, 26 accountsSortValidator,
27 ensureAuthUserOwnsAccountValidator, 27 ensureAuthUserOwnsAccountValidator,
28 ensureCanManageUser,
28 videoChannelsSortValidator, 29 videoChannelsSortValidator,
29 videoChannelStatsValidator, 30 videoChannelStatsValidator,
31 videoChannelSyncsSortValidator,
30 videosSortValidator 32 videosSortValidator
31} from '../../middlewares/validators' 33} from '../../middlewares/validators'
32import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists' 34import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
@@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate'
35import { VideoModel } from '../../models/video/video' 37import { VideoModel } from '../../models/video/video'
36import { VideoChannelModel } from '../../models/video/video-channel' 38import { VideoChannelModel } from '../../models/video/video-channel'
37import { VideoPlaylistModel } from '../../models/video/video-playlist' 39import { VideoPlaylistModel } from '../../models/video/video-playlist'
40import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
38 41
39const accountsRouter = express.Router() 42const accountsRouter = express.Router()
40 43
@@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels',
72 asyncMiddleware(listAccountChannels) 75 asyncMiddleware(listAccountChannels)
73) 76)
74 77
78accountsRouter.get('/:accountName/video-channel-syncs',
79 authenticate,
80 asyncMiddleware(accountNameWithHostGetValidator),
81 ensureCanManageUser,
82 paginationValidator,
83 videoChannelSyncsSortValidator,
84 setDefaultSort,
85 setDefaultPagination,
86 asyncMiddleware(listAccountChannelsSync)
87)
88
75accountsRouter.get('/:accountName/video-playlists', 89accountsRouter.get('/:accountName/video-playlists',
76 optionalAuthenticate, 90 optionalAuthenticate,
77 asyncMiddleware(accountNameWithHostGetValidator), 91 asyncMiddleware(accountNameWithHostGetValidator),
@@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response)
146 return res.json(getFormattedObjects(resultList.data, resultList.total)) 160 return res.json(getFormattedObjects(resultList.data, resultList.total))
147} 161}
148 162
163async function listAccountChannelsSync (req: express.Request, res: express.Response) {
164 const options = {
165 accountId: res.locals.account.id,
166 start: req.query.start,
167 count: req.query.count,
168 sort: req.query.sort,
169 search: req.query.search
170 }
171
172 const resultList = await VideoChannelSyncModel.listByAccountForAPI(options)
173
174 return res.json(getFormattedObjects(resultList.data, resultList.total))
175}
176
149async function listAccountPlaylists (req: express.Request, res: express.Response) { 177async function listAccountPlaylists (req: express.Request, res: express.Response) {
150 const serverActor = await getServerActor() 178 const serverActor = await getServerActor()
151 179
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index ff2fa9d86..f0fb43071 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -273,6 +273,10 @@ function customConfig (): CustomConfig {
273 torrent: { 273 torrent: {
274 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED 274 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
275 } 275 }
276 },
277 videoChannelSynchronization: {
278 enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
279 maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
276 } 280 }
277 }, 281 },
278 trending: { 282 trending: {
diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts
index d1d4ef765..8c8ebd061 100644
--- a/server/controllers/api/index.ts
+++ b/server/controllers/api/index.ts
@@ -20,6 +20,7 @@ import { usersRouter } from './users'
20import { videoChannelRouter } from './video-channel' 20import { videoChannelRouter } from './video-channel'
21import { videoPlaylistRouter } from './video-playlist' 21import { videoPlaylistRouter } from './video-playlist'
22import { videosRouter } from './videos' 22import { videosRouter } from './videos'
23import { videoChannelSyncRouter } from './video-channel-sync'
23 24
24const apiRouter = express.Router() 25const apiRouter = express.Router()
25 26
@@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter)
43apiRouter.use('/users', usersRouter) 44apiRouter.use('/users', usersRouter)
44apiRouter.use('/accounts', accountsRouter) 45apiRouter.use('/accounts', accountsRouter)
45apiRouter.use('/video-channels', videoChannelRouter) 46apiRouter.use('/video-channels', videoChannelRouter)
47apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
46apiRouter.use('/video-playlists', videoPlaylistRouter) 48apiRouter.use('/video-playlists', videoPlaylistRouter)
47apiRouter.use('/videos', videosRouter) 49apiRouter.use('/videos', videosRouter)
48apiRouter.use('/jobs', jobsRouter) 50apiRouter.use('/jobs', jobsRouter)
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index e09510dc3..4e5333782 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models'
7import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' 7import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
8import { UserRight } from '../../../../shared/models/users' 8import { UserRight } from '../../../../shared/models/users'
9import { authenticate, ensureUserHasRight } from '../../../middlewares' 9import { authenticate, ensureUserHasRight } from '../../../middlewares'
10import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
10 11
11const debugRouter = express.Router() 12const debugRouter = express.Router()
12 13
@@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) {
43 const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = { 44 const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
44 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), 45 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
45 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), 46 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
46 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats() 47 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
48 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
47 } 49 }
48 50
49 await processors[body.command]() 51 await processors[body.command]()
diff --git a/server/controllers/api/video-channel-sync.ts b/server/controllers/api/video-channel-sync.ts
new file mode 100644
index 000000000..c2770b8e4
--- /dev/null
+++ b/server/controllers/api/video-channel-sync.ts
@@ -0,0 +1,76 @@
1import express from 'express'
2import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger'
3import { logger } from '@server/helpers/logger'
4import {
5 asyncMiddleware,
6 asyncRetryTransactionMiddleware,
7 authenticate,
8 ensureCanManageChannel as ensureCanManageSyncedChannel,
9 ensureSyncExists,
10 ensureSyncIsEnabled,
11 videoChannelSyncValidator
12} from '@server/middlewares'
13import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
14import { MChannelSyncFormattable } from '@server/types/models'
15import { HttpStatusCode, VideoChannelSyncState } from '@shared/models'
16
17const videoChannelSyncRouter = express.Router()
18const auditLogger = auditLoggerFactory('channel-syncs')
19
20videoChannelSyncRouter.post('/',
21 authenticate,
22 ensureSyncIsEnabled,
23 asyncMiddleware(videoChannelSyncValidator),
24 ensureCanManageSyncedChannel,
25 asyncRetryTransactionMiddleware(createVideoChannelSync)
26)
27
28videoChannelSyncRouter.delete('/:id',
29 authenticate,
30 asyncMiddleware(ensureSyncExists),
31 ensureCanManageSyncedChannel,
32 asyncRetryTransactionMiddleware(removeVideoChannelSync)
33)
34
35export { videoChannelSyncRouter }
36
37// ---------------------------------------------------------------------------
38
39async function createVideoChannelSync (req: express.Request, res: express.Response) {
40 const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({
41 externalChannelUrl: req.body.externalChannelUrl,
42 videoChannelId: req.body.videoChannelId,
43 state: VideoChannelSyncState.WAITING_FIRST_RUN
44 })
45
46 await syncCreated.save()
47 syncCreated.VideoChannel = res.locals.videoChannel
48
49 auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON()))
50
51 logger.info(
52 'Video synchronization for channel "%s" with external channel "%s" created.',
53 syncCreated.VideoChannel.name,
54 syncCreated.externalChannelUrl
55 )
56
57 return res.json({
58 videoChannelSync: syncCreated.toFormattedJSON()
59 })
60}
61
62async function removeVideoChannelSync (req: express.Request, res: express.Response) {
63 const syncInstance = res.locals.videoChannelSync
64
65 await syncInstance.destroy()
66
67 auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON()))
68
69 logger.info(
70 'Video synchronization for channel "%s" with external channel "%s" deleted.',
71 syncInstance.VideoChannel.name,
72 syncInstance.externalChannelUrl
73 )
74
75 return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
76}
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts
index 6b33e894d..89c7181bd 100644
--- a/server/controllers/api/video-channel.ts
+++ b/server/controllers/api/video-channel.ts
@@ -36,7 +36,9 @@ import {
36 videoPlaylistsSortValidator 36 videoPlaylistsSortValidator
37} from '../../middlewares' 37} from '../../middlewares'
38import { 38import {
39 ensureChannelOwnerCanUpload,
39 ensureIsLocalChannel, 40 ensureIsLocalChannel,
41 videoChannelImportVideosValidator,
40 videoChannelsFollowersSortValidator, 42 videoChannelsFollowersSortValidator,
41 videoChannelsListValidator, 43 videoChannelsListValidator,
42 videoChannelsNameWithHostValidator, 44 videoChannelsNameWithHostValidator,
@@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers',
161 asyncMiddleware(listVideoChannelFollowers) 163 asyncMiddleware(listVideoChannelFollowers)
162) 164)
163 165
166videoChannelRouter.post('/:nameWithHost/import-videos',
167 authenticate,
168 asyncMiddleware(videoChannelsNameWithHostValidator),
169 videoChannelImportVideosValidator,
170 ensureIsLocalChannel,
171 ensureCanManageChannel,
172 asyncMiddleware(ensureChannelOwnerCanUpload),
173 asyncMiddleware(importVideosInChannel)
174)
175
164// --------------------------------------------------------------------------- 176// ---------------------------------------------------------------------------
165 177
166export { 178export {
@@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res
404 416
405 return res.json(getFormattedObjects(resultList.data, resultList.total)) 417 return res.json(getFormattedObjects(resultList.data, resultList.total))
406} 418}
419
420async function importVideosInChannel (req: express.Request, res: express.Response) {
421 const { externalChannelUrl } = req.body
422
423 await JobQueue.Instance.createJob({
424 type: 'video-channel-import',
425 payload: {
426 externalChannelUrl,
427 videoChannelId: res.locals.videoChannel.id
428 }
429 })
430
431 logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl)
432
433 return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
434}
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 5a2e1006a..9d7b0260b 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,49 +1,20 @@
1import express from 'express' 1import express from 'express'
2import { move, readFile, remove } from 'fs-extra' 2import { move, readFile } from 'fs-extra'
3import { decode } from 'magnet-uri' 3import { decode } from 'magnet-uri'
4import parseTorrent, { Instance } from 'parse-torrent' 4import parseTorrent, { Instance } from 'parse-torrent'
5import { join } from 'path' 5import { join } from 'path'
6import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' 6import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
7import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' 7import { MThumbnail, MVideoThumbnail } from '@server/types/models'
8import { isResolvingToUnicastOnly } from '@server/helpers/dns' 8import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
9import { Hooks } from '@server/lib/plugins/hooks'
10import { ServerConfigManager } from '@server/lib/server-config-manager'
11import { setVideoTags } from '@server/lib/video'
12import { FilteredModelAttributes } from '@server/types'
13import {
14 MChannelAccountDefault,
15 MThumbnail,
16 MUser,
17 MVideoAccountDefault,
18 MVideoCaption,
19 MVideoTag,
20 MVideoThumbnail,
21 MVideoWithBlacklistLight
22} from '@server/types/models'
23import { MVideoImportFormattable } from '@server/types/models/video/video-import'
24import {
25 HttpStatusCode,
26 ServerErrorCode,
27 ThumbnailType,
28 VideoImportCreate,
29 VideoImportState,
30 VideoPrivacy,
31 VideoState
32} from '@shared/models'
33import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' 9import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
34import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
35import { isArray } from '../../../helpers/custom-validators/misc' 10import { isArray } from '../../../helpers/custom-validators/misc'
36import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' 11import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
37import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
38import { getSecureTorrentName } from '../../../helpers/utils' 13import { getSecureTorrentName } from '../../../helpers/utils'
39import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
40import { CONFIG } from '../../../initializers/config' 14import { CONFIG } from '../../../initializers/config'
41import { MIMETYPES } from '../../../initializers/constants' 15import { MIMETYPES } from '../../../initializers/constants'
42import { sequelizeTypescript } from '../../../initializers/database'
43import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
44import { JobQueue } from '../../../lib/job-queue/job-queue' 16import { JobQueue } from '../../../lib/job-queue/job-queue'
45import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail' 17import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
46import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
47import { 18import {
48 asyncMiddleware, 19 asyncMiddleware,
49 asyncRetryTransactionMiddleware, 20 asyncRetryTransactionMiddleware,
@@ -52,9 +23,6 @@ import {
52 videoImportCancelValidator, 23 videoImportCancelValidator,
53 videoImportDeleteValidator 24 videoImportDeleteValidator
54} from '../../../middlewares' 25} from '../../../middlewares'
55import { VideoModel } from '../../../models/video/video'
56import { VideoCaptionModel } from '../../../models/video/video-caption'
57import { VideoImportModel } from '../../../models/video/video-import'
58 26
59const auditLogger = auditLoggerFactory('video-imports') 27const auditLogger = auditLoggerFactory('video-imports')
60const videoImportsRouter = express.Router() 28const videoImportsRouter = express.Router()
@@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
68 authenticate, 36 authenticate,
69 reqVideoFileImport, 37 reqVideoFileImport,
70 asyncMiddleware(videoImportAddValidator), 38 asyncMiddleware(videoImportAddValidator),
71 asyncRetryTransactionMiddleware(addVideoImport) 39 asyncRetryTransactionMiddleware(handleVideoImport)
72) 40)
73 41
74videoImportsRouter.post('/imports/:id/cancel', 42videoImportsRouter.post('/imports/:id/cancel',
@@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
108 return res.sendStatus(HttpStatusCode.NO_CONTENT_204) 76 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
109} 77}
110 78
111function addVideoImport (req: express.Request, res: express.Response) { 79function handleVideoImport (req: express.Request, res: express.Response) {
112 if (req.body.targetUrl) return addYoutubeDLImport(req, res) 80 if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
113 81
114 const file = req.files?.['torrentfile']?.[0] 82 const file = req.files?.['torrentfile']?.[0]
115 if (req.body.magnetUri || file) return addTorrentImport(req, res, file) 83 if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
116} 84}
117 85
118async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { 86async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
119 const body: VideoImportCreate = req.body 87 const body: VideoImportCreate = req.body
120 const user = res.locals.oauth.token.User 88 const user = res.locals.oauth.token.User
121 89
@@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
135 videoName = result.name 103 videoName = result.name
136 } 104 }
137 105
138 const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName }) 106 const video = await buildVideoFromImport({
107 channelId: res.locals.videoChannel.id,
108 importData: { name: videoName },
109 importDataOverride: body,
110 importType: 'torrent'
111 })
139 112
140 const thumbnailModel = await processThumbnail(req, video) 113 const thumbnailModel = await processThumbnail(req, video)
141 const previewModel = await processPreview(req, video) 114 const previewModel = await processPreview(req, video)
142 115
143 const videoImport = await insertIntoDB({ 116 const videoImport = await insertFromImportIntoDB({
144 video, 117 video,
145 thumbnailModel, 118 thumbnailModel,
146 previewModel, 119 previewModel,
@@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
155 } 128 }
156 }) 129 })
157 130
158 // Create job to import the video 131 const payload: VideoImportPayload = {
159 const payload = {
160 type: torrentfile 132 type: torrentfile
161 ? 'torrent-file' as 'torrent-file' 133 ? 'torrent-file'
162 : 'magnet-uri' as 'magnet-uri', 134 : 'magnet-uri',
163 videoImportId: videoImport.id, 135 videoImportId: videoImport.id,
164 magnetUri 136 preventException: false
165 } 137 }
166 await JobQueue.Instance.createJob({ type: 'video-import', payload }) 138 await JobQueue.Instance.createJob({ type: 'video-import', payload })
167 139
@@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
170 return res.json(videoImport.toFormattedJSON()).end() 142 return res.json(videoImport.toFormattedJSON()).end()
171} 143}
172 144
173async function addYoutubeDLImport (req: express.Request, res: express.Response) { 145function statusFromYtDlImportError (err: YoutubeDlImportError): number {
146 switch (err.code) {
147 case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
148 return HttpStatusCode.FORBIDDEN_403
149
150 case YoutubeDlImportError.CODE.FETCH_ERROR:
151 return HttpStatusCode.BAD_REQUEST_400
152
153 default:
154 return HttpStatusCode.INTERNAL_SERVER_ERROR_500
155 }
156}
157
158async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
174 const body: VideoImportCreate = req.body 159 const body: VideoImportCreate = req.body
175 const targetUrl = body.targetUrl 160 const targetUrl = body.targetUrl
176 const user = res.locals.oauth.token.User 161 const user = res.locals.oauth.token.User
177 162
178 const youtubeDL = new YoutubeDLWrapper(
179 targetUrl,
180 ServerConfigManager.Instance.getEnabledResolutions('vod'),
181 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
182 )
183
184 // Get video infos
185 let youtubeDLInfo: YoutubeDLInfo
186 try { 163 try {
187 youtubeDLInfo = await youtubeDL.getInfoForDownload() 164 const { job, videoImport } = await buildYoutubeDLImport({
165 targetUrl,
166 channel: res.locals.videoChannel,
167 importDataOverride: body,
168 thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
169 previewFilePath: req.files?.['previewfile']?.[0].path,
170 user
171 })
172 await JobQueue.Instance.createJob(job)
173
174 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
175
176 return res.json(videoImport.toFormattedJSON()).end()
188 } catch (err) { 177 } catch (err) {
189 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) 178 logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
190 179
191 return res.fail({ 180 return res.fail({
192 message: 'Cannot fetch remote information of this URL.', 181 message: err.message,
182 status: statusFromYtDlImportError(err),
193 data: { 183 data: {
194 targetUrl 184 targetUrl
195 } 185 }
196 }) 186 })
197 } 187 }
198
199 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
200 return res.fail({
201 status: HttpStatusCode.FORBIDDEN_403,
202 message: 'Cannot use non unicast IP as targetUrl.'
203 })
204 }
205
206 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
207
208 // Process video thumbnail from request.files
209 let thumbnailModel = await processThumbnail(req, video)
210
211 // Process video thumbnail from url if processing from request.files failed
212 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
213 try {
214 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
215 } catch (err) {
216 logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
217 }
218 }
219
220 // Process video preview from request.files
221 let previewModel = await processPreview(req, video)
222
223 // Process video preview from url if processing from request.files failed
224 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
225 try {
226 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
227 } catch (err) {
228 logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
229 }
230 }
231
232 const videoImport = await insertIntoDB({
233 video,
234 thumbnailModel,
235 previewModel,
236 videoChannel: res.locals.videoChannel,
237 tags: body.tags || youtubeDLInfo.tags,
238 user,
239 videoImportAttributes: {
240 targetUrl,
241 state: VideoImportState.PENDING,
242 userId: user.id
243 }
244 })
245
246 // Get video subtitles
247 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
248
249 let fileExt = `.${youtubeDLInfo.ext}`
250 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
251
252 // Create job to import the video
253 const payload = {
254 type: 'youtube-dl' as 'youtube-dl',
255 videoImportId: videoImport.id,
256 fileExt
257 }
258 await JobQueue.Instance.createJob({ type: 'video-import', payload })
259
260 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
261
262 return res.json(videoImport.toFormattedJSON()).end()
263}
264
265async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
266 let videoData = {
267 name: body.name || importData.name || 'Unknown name',
268 remote: false,
269 category: body.category || importData.category,
270 licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
271 language: body.language || importData.language,
272 commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
273 downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
274 waitTranscoding: body.waitTranscoding || false,
275 state: VideoState.TO_IMPORT,
276 nsfw: body.nsfw || importData.nsfw || false,
277 description: body.description || importData.description,
278 support: body.support || null,
279 privacy: body.privacy || VideoPrivacy.PRIVATE,
280 duration: 0, // duration will be set by the import job
281 channelId,
282 originallyPublishedAt: body.originallyPublishedAt
283 ? new Date(body.originallyPublishedAt)
284 : importData.originallyPublishedAt
285 }
286
287 videoData = await Hooks.wrapObject(
288 videoData,
289 body.targetUrl
290 ? 'filter:api.video.import-url.video-attribute.result'
291 : 'filter:api.video.import-torrent.video-attribute.result'
292 )
293
294 const video = new VideoModel(videoData)
295 video.url = getLocalVideoActivityPubUrl(video)
296
297 return video
298} 188}
299 189
300async function processThumbnail (req: express.Request, video: MVideoThumbnail) { 190async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
@@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
329 return undefined 219 return undefined
330} 220}
331 221
332async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
333 try {
334 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
335 } catch (err) {
336 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
337 return undefined
338 }
339}
340
341async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
342 try {
343 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
344 } catch (err) {
345 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
346 return undefined
347 }
348}
349
350async function insertIntoDB (parameters: {
351 video: MVideoThumbnail
352 thumbnailModel: MThumbnail
353 previewModel: MThumbnail
354 videoChannel: MChannelAccountDefault
355 tags: string[]
356 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
357 user: MUser
358}): Promise<MVideoImportFormattable> {
359 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
360
361 const videoImport = await sequelizeTypescript.transaction(async t => {
362 const sequelizeOptions = { transaction: t }
363
364 // Save video object in database
365 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
366 videoCreated.VideoChannel = videoChannel
367
368 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
369 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
370
371 await autoBlacklistVideoIfNeeded({
372 video: videoCreated,
373 user,
374 notify: false,
375 isRemote: false,
376 isNew: true,
377 transaction: t
378 })
379
380 await setVideoTags({ video: videoCreated, tags, transaction: t })
381
382 // Create video import object in database
383 const videoImport = await VideoImportModel.create(
384 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
385 sequelizeOptions
386 ) as MVideoImportFormattable
387 videoImport.Video = videoCreated
388
389 return videoImport
390 })
391
392 return videoImport
393}
394
395async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { 222async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
396 const torrentName = torrentfile.originalname 223 const torrentName = torrentfile.originalname
397 224
@@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
432function extractNameFromArray (name: string | string[]) { 259function extractNameFromArray (name: string | string[]) {
433 return isArray(name) ? name[0] : name 260 return isArray(name) ? name[0] : name
434} 261}
435
436async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
437 try {
438 const subtitles = await youtubeDL.getSubtitles()
439
440 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
441
442 for (const subtitle of subtitles) {
443 if (!await isVTTFileValid(subtitle.path)) {
444 await remove(subtitle.path)
445 continue
446 }
447
448 const videoCaption = new VideoCaptionModel({
449 videoId,
450 language: subtitle.language,
451 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
452 }) as MVideoCaption
453
454 // Move physical file
455 await moveAndProcessCaptionFile(subtitle, videoCaption)
456
457 await sequelizeTypescript.transaction(async t => {
458 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
459 })
460 }
461 } catch (err) {
462 logger.warn('Cannot get video subtitles.', { err })
463 }
464}
465
466async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
467 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
468 const uniqHosts = new Set(hosts)
469
470 for (const h of uniqHosts) {
471 if (await isResolvingToUnicastOnly(h) !== true) {
472 return false
473 }
474 }
475
476 return true
477}