diff options
author | Florent <florent.git@zeteo.me> | 2022-08-10 09:53:39 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-10 09:53:39 +0200 |
commit | 2a491182e483b97afb1b65c908b23cb48d591807 (patch) | |
tree | ec13503216ad72a3ea8f1ce3b659899f8167fb47 /server | |
parent | 06ac128958c489efe1008eeca1df683819bd2f18 (diff) | |
download | PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.gz PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.zst PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.zip |
Channel sync (#5135)
* Add external channel URL for channel update / creation (#754)
* Disallow synchronisation if user has no video quota (#754)
* More constraints serverside (#754)
* Disable sync if server configuration does not allow HTTP import (#754)
* Working version synchronizing videos with a job (#754)
TODO: refactoring, too much code duplication
* More logs and try/catch (#754)
* Fix eslint error (#754)
* WIP: support synchronization time change (#754)
* New frontend #754
* WIP: Create sync front (#754)
* Enhance UI, sync creation form (#754)
* Warning message when HTTP upload is disallowed
* More consistent names (#754)
* Binding Front with API (#754)
* Add a /me API (#754)
* Improve list UI (#754)
* Implement creation and deletion routes (#754)
* Lint (#754)
* Lint again (#754)
* WIP: UI for triggering import existing videos (#754)
* Implement jobs for syncing and importing channels
* Don't sync videos before sync creation + avoid concurrency issue (#754)
* Cleanup (#754)
* Cleanup: OpenAPI + API rework (#754)
* Remove dead code (#754)
* Eslint (#754)
* Revert the mess with whitespaces in constants.ts (#754)
* Some fixes after rebase (#754)
* Several fixes after PR remarks (#754)
* Front + API: Rename video-channels-sync to video-channel-syncs (#754)
* Allow enabling channel sync through UI (#754)
* getChannelInfo (#754)
* Minor fixes: openapi + model + sql (#754)
* Simplified API validators (#754)
* Rename MChannelSync to MChannelSyncChannel (#754)
* Add command for VideoChannelSync (#754)
* Use synchronization.enabled config (#754)
* Check parameters test + some fixes (#754)
* Fix conflict mistake (#754)
* Restrict access to video channel sync list API (#754)
* Start adding unit test for synchronization (#754)
* Continue testing (#754)
* Tests finished + convertion of job to scheduler (#754)
* Add lastSyncAt field (#754)
* Fix externalRemoteUrl sort + creation date not well formatted (#754)
* Small fix (#754)
* Factorize addYoutubeDLImport and buildVideo (#754)
* Check duplicates on channel not on users (#754)
* factorize thumbnail generation (#754)
* Fetch error should return status 400 (#754)
* Separate video-channel-import and video-channel-sync-latest (#754)
* Bump DB migration version after rebase (#754)
* Prettier states in UI table (#754)
* Add DefaultScope in VideoChannelSyncModel (#754)
* Fix audit logs (#754)
* Ensure user can upload when importing channel + minor fixes (#754)
* Mark synchronization as failed on exception + typos (#754)
* Change REST API for importing videos into channel (#754)
* Add option for fully synchronize a chnanel (#754)
* Return a whole sync object on creation to avoid tricks in Front (#754)
* Various remarks (#754)
* Single quotes by default (#754)
* Rename synchronization to video_channel_synchronization
* Add check.latest_videos_count and max_per_user options (#754)
* Better channel rendering in list #754
* Allow sorting with channel name and state (#754)
* Add missing tests for channel imports (#754)
* Prefer using a parent job for channel sync
* Styling
* Client styling
Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server')
49 files changed, 2012 insertions, 339 deletions
diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts index 66cdaab82..7a530cde5 100644 --- a/server/controllers/api/accounts.ts +++ b/server/controllers/api/accounts.ts | |||
@@ -25,8 +25,10 @@ import { | |||
25 | accountsFollowersSortValidator, | 25 | accountsFollowersSortValidator, |
26 | accountsSortValidator, | 26 | accountsSortValidator, |
27 | ensureAuthUserOwnsAccountValidator, | 27 | ensureAuthUserOwnsAccountValidator, |
28 | ensureCanManageUser, | ||
28 | videoChannelsSortValidator, | 29 | videoChannelsSortValidator, |
29 | videoChannelStatsValidator, | 30 | videoChannelStatsValidator, |
31 | videoChannelSyncsSortValidator, | ||
30 | videosSortValidator | 32 | videosSortValidator |
31 | } from '../../middlewares/validators' | 33 | } from '../../middlewares/validators' |
32 | import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists' | 34 | import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists' |
@@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate' | |||
35 | import { VideoModel } from '../../models/video/video' | 37 | import { VideoModel } from '../../models/video/video' |
36 | import { VideoChannelModel } from '../../models/video/video-channel' | 38 | import { VideoChannelModel } from '../../models/video/video-channel' |
37 | import { VideoPlaylistModel } from '../../models/video/video-playlist' | 39 | import { VideoPlaylistModel } from '../../models/video/video-playlist' |
40 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
38 | 41 | ||
39 | const accountsRouter = express.Router() | 42 | const accountsRouter = express.Router() |
40 | 43 | ||
@@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels', | |||
72 | asyncMiddleware(listAccountChannels) | 75 | asyncMiddleware(listAccountChannels) |
73 | ) | 76 | ) |
74 | 77 | ||
78 | accountsRouter.get('/:accountName/video-channel-syncs', | ||
79 | authenticate, | ||
80 | asyncMiddleware(accountNameWithHostGetValidator), | ||
81 | ensureCanManageUser, | ||
82 | paginationValidator, | ||
83 | videoChannelSyncsSortValidator, | ||
84 | setDefaultSort, | ||
85 | setDefaultPagination, | ||
86 | asyncMiddleware(listAccountChannelsSync) | ||
87 | ) | ||
88 | |||
75 | accountsRouter.get('/:accountName/video-playlists', | 89 | accountsRouter.get('/:accountName/video-playlists', |
76 | optionalAuthenticate, | 90 | optionalAuthenticate, |
77 | asyncMiddleware(accountNameWithHostGetValidator), | 91 | asyncMiddleware(accountNameWithHostGetValidator), |
@@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response) | |||
146 | return res.json(getFormattedObjects(resultList.data, resultList.total)) | 160 | return res.json(getFormattedObjects(resultList.data, resultList.total)) |
147 | } | 161 | } |
148 | 162 | ||
163 | async function listAccountChannelsSync (req: express.Request, res: express.Response) { | ||
164 | const options = { | ||
165 | accountId: res.locals.account.id, | ||
166 | start: req.query.start, | ||
167 | count: req.query.count, | ||
168 | sort: req.query.sort, | ||
169 | search: req.query.search | ||
170 | } | ||
171 | |||
172 | const resultList = await VideoChannelSyncModel.listByAccountForAPI(options) | ||
173 | |||
174 | return res.json(getFormattedObjects(resultList.data, resultList.total)) | ||
175 | } | ||
176 | |||
149 | async function listAccountPlaylists (req: express.Request, res: express.Response) { | 177 | async function listAccountPlaylists (req: express.Request, res: express.Response) { |
150 | const serverActor = await getServerActor() | 178 | const serverActor = await getServerActor() |
151 | 179 | ||
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index ff2fa9d86..f0fb43071 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts | |||
@@ -273,6 +273,10 @@ function customConfig (): CustomConfig { | |||
273 | torrent: { | 273 | torrent: { |
274 | enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED | 274 | enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED |
275 | } | 275 | } |
276 | }, | ||
277 | videoChannelSynchronization: { | ||
278 | enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED, | ||
279 | maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER | ||
276 | } | 280 | } |
277 | }, | 281 | }, |
278 | trending: { | 282 | trending: { |
diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index d1d4ef765..8c8ebd061 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts | |||
@@ -20,6 +20,7 @@ import { usersRouter } from './users' | |||
20 | import { videoChannelRouter } from './video-channel' | 20 | import { videoChannelRouter } from './video-channel' |
21 | import { videoPlaylistRouter } from './video-playlist' | 21 | import { videoPlaylistRouter } from './video-playlist' |
22 | import { videosRouter } from './videos' | 22 | import { videosRouter } from './videos' |
23 | import { videoChannelSyncRouter } from './video-channel-sync' | ||
23 | 24 | ||
24 | const apiRouter = express.Router() | 25 | const apiRouter = express.Router() |
25 | 26 | ||
@@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter) | |||
43 | apiRouter.use('/users', usersRouter) | 44 | apiRouter.use('/users', usersRouter) |
44 | apiRouter.use('/accounts', accountsRouter) | 45 | apiRouter.use('/accounts', accountsRouter) |
45 | apiRouter.use('/video-channels', videoChannelRouter) | 46 | apiRouter.use('/video-channels', videoChannelRouter) |
47 | apiRouter.use('/video-channel-syncs', videoChannelSyncRouter) | ||
46 | apiRouter.use('/video-playlists', videoPlaylistRouter) | 48 | apiRouter.use('/video-playlists', videoPlaylistRouter) |
47 | apiRouter.use('/videos', videosRouter) | 49 | apiRouter.use('/videos', videosRouter) |
48 | apiRouter.use('/jobs', jobsRouter) | 50 | apiRouter.use('/jobs', jobsRouter) |
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index e09510dc3..4e5333782 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts | |||
@@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models' | |||
7 | import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' | 7 | import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' |
8 | import { UserRight } from '../../../../shared/models/users' | 8 | import { UserRight } from '../../../../shared/models/users' |
9 | import { authenticate, ensureUserHasRight } from '../../../middlewares' | 9 | import { authenticate, ensureUserHasRight } from '../../../middlewares' |
10 | import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler' | ||
10 | 11 | ||
11 | const debugRouter = express.Router() | 12 | const debugRouter = express.Router() |
12 | 13 | ||
@@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) { | |||
43 | const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = { | 44 | const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = { |
44 | 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), | 45 | 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), |
45 | 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), | 46 | 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), |
46 | 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats() | 47 | 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), |
48 | 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute() | ||
47 | } | 49 | } |
48 | 50 | ||
49 | await processors[body.command]() | 51 | await processors[body.command]() |
diff --git a/server/controllers/api/video-channel-sync.ts b/server/controllers/api/video-channel-sync.ts new file mode 100644 index 000000000..c2770b8e4 --- /dev/null +++ b/server/controllers/api/video-channel-sync.ts | |||
@@ -0,0 +1,76 @@ | |||
1 | import express from 'express' | ||
2 | import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { | ||
5 | asyncMiddleware, | ||
6 | asyncRetryTransactionMiddleware, | ||
7 | authenticate, | ||
8 | ensureCanManageChannel as ensureCanManageSyncedChannel, | ||
9 | ensureSyncExists, | ||
10 | ensureSyncIsEnabled, | ||
11 | videoChannelSyncValidator | ||
12 | } from '@server/middlewares' | ||
13 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
14 | import { MChannelSyncFormattable } from '@server/types/models' | ||
15 | import { HttpStatusCode, VideoChannelSyncState } from '@shared/models' | ||
16 | |||
17 | const videoChannelSyncRouter = express.Router() | ||
18 | const auditLogger = auditLoggerFactory('channel-syncs') | ||
19 | |||
20 | videoChannelSyncRouter.post('/', | ||
21 | authenticate, | ||
22 | ensureSyncIsEnabled, | ||
23 | asyncMiddleware(videoChannelSyncValidator), | ||
24 | ensureCanManageSyncedChannel, | ||
25 | asyncRetryTransactionMiddleware(createVideoChannelSync) | ||
26 | ) | ||
27 | |||
28 | videoChannelSyncRouter.delete('/:id', | ||
29 | authenticate, | ||
30 | asyncMiddleware(ensureSyncExists), | ||
31 | ensureCanManageSyncedChannel, | ||
32 | asyncRetryTransactionMiddleware(removeVideoChannelSync) | ||
33 | ) | ||
34 | |||
35 | export { videoChannelSyncRouter } | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | |||
39 | async function createVideoChannelSync (req: express.Request, res: express.Response) { | ||
40 | const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({ | ||
41 | externalChannelUrl: req.body.externalChannelUrl, | ||
42 | videoChannelId: req.body.videoChannelId, | ||
43 | state: VideoChannelSyncState.WAITING_FIRST_RUN | ||
44 | }) | ||
45 | |||
46 | await syncCreated.save() | ||
47 | syncCreated.VideoChannel = res.locals.videoChannel | ||
48 | |||
49 | auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON())) | ||
50 | |||
51 | logger.info( | ||
52 | 'Video synchronization for channel "%s" with external channel "%s" created.', | ||
53 | syncCreated.VideoChannel.name, | ||
54 | syncCreated.externalChannelUrl | ||
55 | ) | ||
56 | |||
57 | return res.json({ | ||
58 | videoChannelSync: syncCreated.toFormattedJSON() | ||
59 | }) | ||
60 | } | ||
61 | |||
62 | async function removeVideoChannelSync (req: express.Request, res: express.Response) { | ||
63 | const syncInstance = res.locals.videoChannelSync | ||
64 | |||
65 | await syncInstance.destroy() | ||
66 | |||
67 | auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON())) | ||
68 | |||
69 | logger.info( | ||
70 | 'Video synchronization for channel "%s" with external channel "%s" deleted.', | ||
71 | syncInstance.VideoChannel.name, | ||
72 | syncInstance.externalChannelUrl | ||
73 | ) | ||
74 | |||
75 | return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() | ||
76 | } | ||
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 6b33e894d..89c7181bd 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts | |||
@@ -36,7 +36,9 @@ import { | |||
36 | videoPlaylistsSortValidator | 36 | videoPlaylistsSortValidator |
37 | } from '../../middlewares' | 37 | } from '../../middlewares' |
38 | import { | 38 | import { |
39 | ensureChannelOwnerCanUpload, | ||
39 | ensureIsLocalChannel, | 40 | ensureIsLocalChannel, |
41 | videoChannelImportVideosValidator, | ||
40 | videoChannelsFollowersSortValidator, | 42 | videoChannelsFollowersSortValidator, |
41 | videoChannelsListValidator, | 43 | videoChannelsListValidator, |
42 | videoChannelsNameWithHostValidator, | 44 | videoChannelsNameWithHostValidator, |
@@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers', | |||
161 | asyncMiddleware(listVideoChannelFollowers) | 163 | asyncMiddleware(listVideoChannelFollowers) |
162 | ) | 164 | ) |
163 | 165 | ||
166 | videoChannelRouter.post('/:nameWithHost/import-videos', | ||
167 | authenticate, | ||
168 | asyncMiddleware(videoChannelsNameWithHostValidator), | ||
169 | videoChannelImportVideosValidator, | ||
170 | ensureIsLocalChannel, | ||
171 | ensureCanManageChannel, | ||
172 | asyncMiddleware(ensureChannelOwnerCanUpload), | ||
173 | asyncMiddleware(importVideosInChannel) | ||
174 | ) | ||
175 | |||
164 | // --------------------------------------------------------------------------- | 176 | // --------------------------------------------------------------------------- |
165 | 177 | ||
166 | export { | 178 | export { |
@@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res | |||
404 | 416 | ||
405 | return res.json(getFormattedObjects(resultList.data, resultList.total)) | 417 | return res.json(getFormattedObjects(resultList.data, resultList.total)) |
406 | } | 418 | } |
419 | |||
420 | async function importVideosInChannel (req: express.Request, res: express.Response) { | ||
421 | const { externalChannelUrl } = req.body | ||
422 | |||
423 | await JobQueue.Instance.createJob({ | ||
424 | type: 'video-channel-import', | ||
425 | payload: { | ||
426 | externalChannelUrl, | ||
427 | videoChannelId: res.locals.videoChannel.id | ||
428 | } | ||
429 | }) | ||
430 | |||
431 | logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl) | ||
432 | |||
433 | return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() | ||
434 | } | ||
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 5a2e1006a..9d7b0260b 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -1,49 +1,20 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { move, readFile, remove } from 'fs-extra' | 2 | import { move, readFile } from 'fs-extra' |
3 | import { decode } from 'magnet-uri' | 3 | import { decode } from 'magnet-uri' |
4 | import parseTorrent, { Instance } from 'parse-torrent' | 4 | import parseTorrent, { Instance } from 'parse-torrent' |
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' | 6 | import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import' |
7 | import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' | 7 | import { MThumbnail, MVideoThumbnail } from '@server/types/models' |
8 | import { isResolvingToUnicastOnly } from '@server/helpers/dns' | 8 | import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models' |
9 | import { Hooks } from '@server/lib/plugins/hooks' | ||
10 | import { ServerConfigManager } from '@server/lib/server-config-manager' | ||
11 | import { setVideoTags } from '@server/lib/video' | ||
12 | import { FilteredModelAttributes } from '@server/types' | ||
13 | import { | ||
14 | MChannelAccountDefault, | ||
15 | MThumbnail, | ||
16 | MUser, | ||
17 | MVideoAccountDefault, | ||
18 | MVideoCaption, | ||
19 | MVideoTag, | ||
20 | MVideoThumbnail, | ||
21 | MVideoWithBlacklistLight | ||
22 | } from '@server/types/models' | ||
23 | import { MVideoImportFormattable } from '@server/types/models/video/video-import' | ||
24 | import { | ||
25 | HttpStatusCode, | ||
26 | ServerErrorCode, | ||
27 | ThumbnailType, | ||
28 | VideoImportCreate, | ||
29 | VideoImportState, | ||
30 | VideoPrivacy, | ||
31 | VideoState | ||
32 | } from '@shared/models' | ||
33 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' | 9 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' |
34 | import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' | ||
35 | import { isArray } from '../../../helpers/custom-validators/misc' | 10 | import { isArray } from '../../../helpers/custom-validators/misc' |
36 | import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' | 11 | import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' |
37 | import { logger } from '../../../helpers/logger' | 12 | import { logger } from '../../../helpers/logger' |
38 | import { getSecureTorrentName } from '../../../helpers/utils' | 13 | import { getSecureTorrentName } from '../../../helpers/utils' |
39 | import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl' | ||
40 | import { CONFIG } from '../../../initializers/config' | 14 | import { CONFIG } from '../../../initializers/config' |
41 | import { MIMETYPES } from '../../../initializers/constants' | 15 | import { MIMETYPES } from '../../../initializers/constants' |
42 | import { sequelizeTypescript } from '../../../initializers/database' | ||
43 | import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url' | ||
44 | import { JobQueue } from '../../../lib/job-queue/job-queue' | 16 | import { JobQueue } from '../../../lib/job-queue/job-queue' |
45 | import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail' | 17 | import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail' |
46 | import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' | ||
47 | import { | 18 | import { |
48 | asyncMiddleware, | 19 | asyncMiddleware, |
49 | asyncRetryTransactionMiddleware, | 20 | asyncRetryTransactionMiddleware, |
@@ -52,9 +23,6 @@ import { | |||
52 | videoImportCancelValidator, | 23 | videoImportCancelValidator, |
53 | videoImportDeleteValidator | 24 | videoImportDeleteValidator |
54 | } from '../../../middlewares' | 25 | } from '../../../middlewares' |
55 | import { VideoModel } from '../../../models/video/video' | ||
56 | import { VideoCaptionModel } from '../../../models/video/video-caption' | ||
57 | import { VideoImportModel } from '../../../models/video/video-import' | ||
58 | 26 | ||
59 | const auditLogger = auditLoggerFactory('video-imports') | 27 | const auditLogger = auditLoggerFactory('video-imports') |
60 | const videoImportsRouter = express.Router() | 28 | const videoImportsRouter = express.Router() |
@@ -68,7 +36,7 @@ videoImportsRouter.post('/imports', | |||
68 | authenticate, | 36 | authenticate, |
69 | reqVideoFileImport, | 37 | reqVideoFileImport, |
70 | asyncMiddleware(videoImportAddValidator), | 38 | asyncMiddleware(videoImportAddValidator), |
71 | asyncRetryTransactionMiddleware(addVideoImport) | 39 | asyncRetryTransactionMiddleware(handleVideoImport) |
72 | ) | 40 | ) |
73 | 41 | ||
74 | videoImportsRouter.post('/imports/:id/cancel', | 42 | videoImportsRouter.post('/imports/:id/cancel', |
@@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) { | |||
108 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | 76 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) |
109 | } | 77 | } |
110 | 78 | ||
111 | function addVideoImport (req: express.Request, res: express.Response) { | 79 | function handleVideoImport (req: express.Request, res: express.Response) { |
112 | if (req.body.targetUrl) return addYoutubeDLImport(req, res) | 80 | if (req.body.targetUrl) return handleYoutubeDlImport(req, res) |
113 | 81 | ||
114 | const file = req.files?.['torrentfile']?.[0] | 82 | const file = req.files?.['torrentfile']?.[0] |
115 | if (req.body.magnetUri || file) return addTorrentImport(req, res, file) | 83 | if (req.body.magnetUri || file) return handleTorrentImport(req, res, file) |
116 | } | 84 | } |
117 | 85 | ||
118 | async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { | 86 | async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { |
119 | const body: VideoImportCreate = req.body | 87 | const body: VideoImportCreate = req.body |
120 | const user = res.locals.oauth.token.User | 88 | const user = res.locals.oauth.token.User |
121 | 89 | ||
@@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to | |||
135 | videoName = result.name | 103 | videoName = result.name |
136 | } | 104 | } |
137 | 105 | ||
138 | const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName }) | 106 | const video = await buildVideoFromImport({ |
107 | channelId: res.locals.videoChannel.id, | ||
108 | importData: { name: videoName }, | ||
109 | importDataOverride: body, | ||
110 | importType: 'torrent' | ||
111 | }) | ||
139 | 112 | ||
140 | const thumbnailModel = await processThumbnail(req, video) | 113 | const thumbnailModel = await processThumbnail(req, video) |
141 | const previewModel = await processPreview(req, video) | 114 | const previewModel = await processPreview(req, video) |
142 | 115 | ||
143 | const videoImport = await insertIntoDB({ | 116 | const videoImport = await insertFromImportIntoDB({ |
144 | video, | 117 | video, |
145 | thumbnailModel, | 118 | thumbnailModel, |
146 | previewModel, | 119 | previewModel, |
@@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to | |||
155 | } | 128 | } |
156 | }) | 129 | }) |
157 | 130 | ||
158 | // Create job to import the video | 131 | const payload: VideoImportPayload = { |
159 | const payload = { | ||
160 | type: torrentfile | 132 | type: torrentfile |
161 | ? 'torrent-file' as 'torrent-file' | 133 | ? 'torrent-file' |
162 | : 'magnet-uri' as 'magnet-uri', | 134 | : 'magnet-uri', |
163 | videoImportId: videoImport.id, | 135 | videoImportId: videoImport.id, |
164 | magnetUri | 136 | preventException: false |
165 | } | 137 | } |
166 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | 138 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) |
167 | 139 | ||
@@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to | |||
170 | return res.json(videoImport.toFormattedJSON()).end() | 142 | return res.json(videoImport.toFormattedJSON()).end() |
171 | } | 143 | } |
172 | 144 | ||
173 | async function addYoutubeDLImport (req: express.Request, res: express.Response) { | 145 | function statusFromYtDlImportError (err: YoutubeDlImportError): number { |
146 | switch (err.code) { | ||
147 | case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL: | ||
148 | return HttpStatusCode.FORBIDDEN_403 | ||
149 | |||
150 | case YoutubeDlImportError.CODE.FETCH_ERROR: | ||
151 | return HttpStatusCode.BAD_REQUEST_400 | ||
152 | |||
153 | default: | ||
154 | return HttpStatusCode.INTERNAL_SERVER_ERROR_500 | ||
155 | } | ||
156 | } | ||
157 | |||
158 | async function handleYoutubeDlImport (req: express.Request, res: express.Response) { | ||
174 | const body: VideoImportCreate = req.body | 159 | const body: VideoImportCreate = req.body |
175 | const targetUrl = body.targetUrl | 160 | const targetUrl = body.targetUrl |
176 | const user = res.locals.oauth.token.User | 161 | const user = res.locals.oauth.token.User |
177 | 162 | ||
178 | const youtubeDL = new YoutubeDLWrapper( | ||
179 | targetUrl, | ||
180 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | ||
181 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
182 | ) | ||
183 | |||
184 | // Get video infos | ||
185 | let youtubeDLInfo: YoutubeDLInfo | ||
186 | try { | 163 | try { |
187 | youtubeDLInfo = await youtubeDL.getInfoForDownload() | 164 | const { job, videoImport } = await buildYoutubeDLImport({ |
165 | targetUrl, | ||
166 | channel: res.locals.videoChannel, | ||
167 | importDataOverride: body, | ||
168 | thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path, | ||
169 | previewFilePath: req.files?.['previewfile']?.[0].path, | ||
170 | user | ||
171 | }) | ||
172 | await JobQueue.Instance.createJob(job) | ||
173 | |||
174 | auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) | ||
175 | |||
176 | return res.json(videoImport.toFormattedJSON()).end() | ||
188 | } catch (err) { | 177 | } catch (err) { |
189 | logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) | 178 | logger.error('An error occurred while importing the video %s. ', targetUrl, { err }) |
190 | 179 | ||
191 | return res.fail({ | 180 | return res.fail({ |
192 | message: 'Cannot fetch remote information of this URL.', | 181 | message: err.message, |
182 | status: statusFromYtDlImportError(err), | ||
193 | data: { | 183 | data: { |
194 | targetUrl | 184 | targetUrl |
195 | } | 185 | } |
196 | }) | 186 | }) |
197 | } | 187 | } |
198 | |||
199 | if (!await hasUnicastURLsOnly(youtubeDLInfo)) { | ||
200 | return res.fail({ | ||
201 | status: HttpStatusCode.FORBIDDEN_403, | ||
202 | message: 'Cannot use non unicast IP as targetUrl.' | ||
203 | }) | ||
204 | } | ||
205 | |||
206 | const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) | ||
207 | |||
208 | // Process video thumbnail from request.files | ||
209 | let thumbnailModel = await processThumbnail(req, video) | ||
210 | |||
211 | // Process video thumbnail from url if processing from request.files failed | ||
212 | if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) { | ||
213 | try { | ||
214 | thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video) | ||
215 | } catch (err) { | ||
216 | logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err }) | ||
217 | } | ||
218 | } | ||
219 | |||
220 | // Process video preview from request.files | ||
221 | let previewModel = await processPreview(req, video) | ||
222 | |||
223 | // Process video preview from url if processing from request.files failed | ||
224 | if (!previewModel && youtubeDLInfo.thumbnailUrl) { | ||
225 | try { | ||
226 | previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video) | ||
227 | } catch (err) { | ||
228 | logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err }) | ||
229 | } | ||
230 | } | ||
231 | |||
232 | const videoImport = await insertIntoDB({ | ||
233 | video, | ||
234 | thumbnailModel, | ||
235 | previewModel, | ||
236 | videoChannel: res.locals.videoChannel, | ||
237 | tags: body.tags || youtubeDLInfo.tags, | ||
238 | user, | ||
239 | videoImportAttributes: { | ||
240 | targetUrl, | ||
241 | state: VideoImportState.PENDING, | ||
242 | userId: user.id | ||
243 | } | ||
244 | }) | ||
245 | |||
246 | // Get video subtitles | ||
247 | await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) | ||
248 | |||
249 | let fileExt = `.${youtubeDLInfo.ext}` | ||
250 | if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' | ||
251 | |||
252 | // Create job to import the video | ||
253 | const payload = { | ||
254 | type: 'youtube-dl' as 'youtube-dl', | ||
255 | videoImportId: videoImport.id, | ||
256 | fileExt | ||
257 | } | ||
258 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | ||
259 | |||
260 | auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) | ||
261 | |||
262 | return res.json(videoImport.toFormattedJSON()).end() | ||
263 | } | ||
264 | |||
265 | async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> { | ||
266 | let videoData = { | ||
267 | name: body.name || importData.name || 'Unknown name', | ||
268 | remote: false, | ||
269 | category: body.category || importData.category, | ||
270 | licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, | ||
271 | language: body.language || importData.language, | ||
272 | commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, | ||
273 | downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, | ||
274 | waitTranscoding: body.waitTranscoding || false, | ||
275 | state: VideoState.TO_IMPORT, | ||
276 | nsfw: body.nsfw || importData.nsfw || false, | ||
277 | description: body.description || importData.description, | ||
278 | support: body.support || null, | ||
279 | privacy: body.privacy || VideoPrivacy.PRIVATE, | ||
280 | duration: 0, // duration will be set by the import job | ||
281 | channelId, | ||
282 | originallyPublishedAt: body.originallyPublishedAt | ||
283 | ? new Date(body.originallyPublishedAt) | ||
284 | : importData.originallyPublishedAt | ||
285 | } | ||
286 | |||
287 | videoData = await Hooks.wrapObject( | ||
288 | videoData, | ||
289 | body.targetUrl | ||
290 | ? 'filter:api.video.import-url.video-attribute.result' | ||
291 | : 'filter:api.video.import-torrent.video-attribute.result' | ||
292 | ) | ||
293 | |||
294 | const video = new VideoModel(videoData) | ||
295 | video.url = getLocalVideoActivityPubUrl(video) | ||
296 | |||
297 | return video | ||
298 | } | 188 | } |
299 | 189 | ||
300 | async function processThumbnail (req: express.Request, video: MVideoThumbnail) { | 190 | async function processThumbnail (req: express.Request, video: MVideoThumbnail) { |
@@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr | |||
329 | return undefined | 219 | return undefined |
330 | } | 220 | } |
331 | 221 | ||
332 | async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) { | ||
333 | try { | ||
334 | return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE }) | ||
335 | } catch (err) { | ||
336 | logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err }) | ||
337 | return undefined | ||
338 | } | ||
339 | } | ||
340 | |||
341 | async function processPreviewFromUrl (url: string, video: MVideoThumbnail) { | ||
342 | try { | ||
343 | return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW }) | ||
344 | } catch (err) { | ||
345 | logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err }) | ||
346 | return undefined | ||
347 | } | ||
348 | } | ||
349 | |||
350 | async function insertIntoDB (parameters: { | ||
351 | video: MVideoThumbnail | ||
352 | thumbnailModel: MThumbnail | ||
353 | previewModel: MThumbnail | ||
354 | videoChannel: MChannelAccountDefault | ||
355 | tags: string[] | ||
356 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> | ||
357 | user: MUser | ||
358 | }): Promise<MVideoImportFormattable> { | ||
359 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters | ||
360 | |||
361 | const videoImport = await sequelizeTypescript.transaction(async t => { | ||
362 | const sequelizeOptions = { transaction: t } | ||
363 | |||
364 | // Save video object in database | ||
365 | const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) | ||
366 | videoCreated.VideoChannel = videoChannel | ||
367 | |||
368 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | ||
369 | if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) | ||
370 | |||
371 | await autoBlacklistVideoIfNeeded({ | ||
372 | video: videoCreated, | ||
373 | user, | ||
374 | notify: false, | ||
375 | isRemote: false, | ||
376 | isNew: true, | ||
377 | transaction: t | ||
378 | }) | ||
379 | |||
380 | await setVideoTags({ video: videoCreated, tags, transaction: t }) | ||
381 | |||
382 | // Create video import object in database | ||
383 | const videoImport = await VideoImportModel.create( | ||
384 | Object.assign({ videoId: videoCreated.id }, videoImportAttributes), | ||
385 | sequelizeOptions | ||
386 | ) as MVideoImportFormattable | ||
387 | videoImport.Video = videoCreated | ||
388 | |||
389 | return videoImport | ||
390 | }) | ||
391 | |||
392 | return videoImport | ||
393 | } | ||
394 | |||
395 | async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { | 222 | async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { |
396 | const torrentName = torrentfile.originalname | 223 | const torrentName = torrentfile.originalname |
397 | 224 | ||
@@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) { | |||
432 | function extractNameFromArray (name: string | string[]) { | 259 | function extractNameFromArray (name: string | string[]) { |
433 | return isArray(name) ? name[0] : name | 260 | return isArray(name) ? name[0] : name |
434 | } | 261 | } |
435 | |||
436 | async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { | ||
437 | try { | ||
438 | const subtitles = await youtubeDL.getSubtitles() | ||
439 | |||
440 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) | ||
441 | |||
442 | for (const subtitle of subtitles) { | ||
443 | if (!await isVTTFileValid(subtitle.path)) { | ||
444 | await remove(subtitle.path) | ||
445 | continue | ||
446 | } | ||
447 | |||
448 | const videoCaption = new VideoCaptionModel({ | ||
449 | videoId, | ||
450 | language: subtitle.language, | ||
451 | filename: VideoCaptionModel.generateCaptionName(subtitle.language) | ||
452 | }) as MVideoCaption | ||
453 | |||
454 | // Move physical file | ||
455 | await moveAndProcessCaptionFile(subtitle, videoCaption) | ||
456 | |||
457 | await sequelizeTypescript.transaction(async t => { | ||
458 | await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) | ||
459 | }) | ||
460 | } | ||
461 | } catch (err) { | ||
462 | logger.warn('Cannot get video subtitles.', { err }) | ||
463 | } | ||
464 | } | ||
465 | |||
466 | async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { | ||
467 | const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) | ||
468 | const uniqHosts = new Set(hosts) | ||
469 | |||
470 | for (const h of uniqHosts) { | ||
471 | if (await isResolvingToUnicastOnly(h) !== true) { | ||
472 | return false | ||
473 | } | ||
474 | } | ||
475 | |||
476 | return true | ||
477 | } | ||
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index 076b7f11d..7e8a03e8f 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts | |||
@@ -5,7 +5,7 @@ import { chain } from 'lodash' | |||
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { addColors, config, createLogger, format, transports } from 'winston' | 6 | import { addColors, config, createLogger, format, transports } from 'winston' |
7 | import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' | 7 | import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' |
8 | import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models' | 8 | import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models' |
9 | import { CONFIG } from '../initializers/config' | 9 | import { CONFIG } from '../initializers/config' |
10 | import { jsonLoggerFormat, labelFormatter } from './logger' | 10 | import { jsonLoggerFormat, labelFormatter } from './logger' |
11 | 11 | ||
@@ -260,6 +260,18 @@ class CustomConfigAuditView extends EntityAuditView { | |||
260 | } | 260 | } |
261 | } | 261 | } |
262 | 262 | ||
263 | const channelSyncKeysToKeep = [ | ||
264 | 'id', | ||
265 | 'externalChannelUrl', | ||
266 | 'channel-id', | ||
267 | 'channel-name' | ||
268 | ] | ||
269 | class VideoChannelSyncAuditView extends EntityAuditView { | ||
270 | constructor (channelSync: VideoChannelSync) { | ||
271 | super(channelSyncKeysToKeep, 'channelSync', channelSync) | ||
272 | } | ||
273 | } | ||
274 | |||
263 | export { | 275 | export { |
264 | getAuditIdFromRes, | 276 | getAuditIdFromRes, |
265 | 277 | ||
@@ -270,5 +282,6 @@ export { | |||
270 | UserAuditView, | 282 | UserAuditView, |
271 | VideoAuditView, | 283 | VideoAuditView, |
272 | AbuseAuditView, | 284 | AbuseAuditView, |
273 | CustomConfigAuditView | 285 | CustomConfigAuditView, |
286 | VideoChannelSyncAuditView | ||
274 | } | 287 | } |
diff --git a/server/helpers/custom-validators/video-channel-syncs.ts b/server/helpers/custom-validators/video-channel-syncs.ts new file mode 100644 index 000000000..c5a9afa96 --- /dev/null +++ b/server/helpers/custom-validators/video-channel-syncs.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants' | ||
2 | import { exists } from './misc' | ||
3 | |||
4 | export function isVideoChannelSyncStateValid (value: any) { | ||
5 | return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined | ||
6 | } | ||
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts index 13c990a1e..5a87b99b4 100644 --- a/server/helpers/youtube-dl/youtube-dl-cli.ts +++ b/server/helpers/youtube-dl/youtube-dl-cli.ts | |||
@@ -87,6 +87,7 @@ export class YoutubeDLCLI { | |||
87 | return result.concat([ | 87 | return result.concat([ |
88 | 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', | 88 | 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', |
89 | 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats | 89 | 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats |
90 | 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', | ||
90 | 'best' // Ultimate fallback | 91 | 'best' // Ultimate fallback |
91 | ]).join('/') | 92 | ]).join('/') |
92 | } | 93 | } |
@@ -103,11 +104,14 @@ export class YoutubeDLCLI { | |||
103 | timeout?: number | 104 | timeout?: number |
104 | additionalYoutubeDLArgs?: string[] | 105 | additionalYoutubeDLArgs?: string[] |
105 | }) { | 106 | }) { |
107 | let args = options.additionalYoutubeDLArgs || [] | ||
108 | args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ]) | ||
109 | |||
106 | return this.run({ | 110 | return this.run({ |
107 | url: options.url, | 111 | url: options.url, |
108 | processOptions: options.processOptions, | 112 | processOptions: options.processOptions, |
109 | timeout: options.timeout, | 113 | timeout: options.timeout, |
110 | args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ]) | 114 | args |
111 | }) | 115 | }) |
112 | } | 116 | } |
113 | 117 | ||
@@ -129,6 +133,25 @@ export class YoutubeDLCLI { | |||
129 | : info | 133 | : info |
130 | } | 134 | } |
131 | 135 | ||
136 | getListInfo (options: { | ||
137 | url: string | ||
138 | latestVideosCount?: number | ||
139 | processOptions: execa.NodeOptions | ||
140 | }): Promise<{ upload_date: string, webpage_url: string }[]> { | ||
141 | const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ] | ||
142 | |||
143 | if (options.latestVideosCount !== undefined) { | ||
144 | additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) | ||
145 | } | ||
146 | |||
147 | return this.getInfo({ | ||
148 | url: options.url, | ||
149 | format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), | ||
150 | processOptions: options.processOptions, | ||
151 | additionalYoutubeDLArgs | ||
152 | }) | ||
153 | } | ||
154 | |||
132 | async getSubs (options: { | 155 | async getSubs (options: { |
133 | url: string | 156 | url: string |
134 | format: 'vtt' | 157 | format: 'vtt' |
@@ -175,7 +198,7 @@ export class YoutubeDLCLI { | |||
175 | 198 | ||
176 | const output = await subProcess | 199 | const output = await subProcess |
177 | 200 | ||
178 | logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() }) | 201 | logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() }) |
179 | 202 | ||
180 | return output.stdout | 203 | return output.stdout |
181 | ? output.stdout.trim().split(/\r?\n/) | 204 | ? output.stdout.trim().split(/\r?\n/) |
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts index 71572f292..303e4051f 100644 --- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts +++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts | |||
@@ -13,6 +13,7 @@ type YoutubeDLInfo = { | |||
13 | thumbnailUrl?: string | 13 | thumbnailUrl?: string |
14 | ext?: string | 14 | ext?: string |
15 | originallyPublishedAt?: Date | 15 | originallyPublishedAt?: Date |
16 | webpageUrl?: string | ||
16 | 17 | ||
17 | urls?: string[] | 18 | urls?: string[] |
18 | } | 19 | } |
@@ -81,7 +82,8 @@ class YoutubeDLInfoBuilder { | |||
81 | thumbnailUrl: obj.thumbnail || undefined, | 82 | thumbnailUrl: obj.thumbnail || undefined, |
82 | urls: this.buildAvailableUrl(obj), | 83 | urls: this.buildAvailableUrl(obj), |
83 | originallyPublishedAt: this.buildOriginallyPublishedAt(obj), | 84 | originallyPublishedAt: this.buildOriginallyPublishedAt(obj), |
84 | ext: obj.ext | 85 | ext: obj.ext, |
86 | webpageUrl: obj.webpage_url | ||
85 | } | 87 | } |
86 | } | 88 | } |
87 | 89 | ||
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts index 176cf3b69..7cd5e3310 100644 --- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts +++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts | |||
@@ -46,6 +46,24 @@ class YoutubeDLWrapper { | |||
46 | return infoBuilder.getInfo() | 46 | return infoBuilder.getInfo() |
47 | } | 47 | } |
48 | 48 | ||
49 | async getInfoForListImport (options: { | ||
50 | latestVideosCount?: number | ||
51 | }) { | ||
52 | const youtubeDL = await YoutubeDLCLI.safeGet() | ||
53 | |||
54 | const list = await youtubeDL.getListInfo({ | ||
55 | url: this.url, | ||
56 | latestVideosCount: options.latestVideosCount, | ||
57 | processOptions | ||
58 | }) | ||
59 | |||
60 | return list.map(info => { | ||
61 | const infoBuilder = new YoutubeDLInfoBuilder(info) | ||
62 | |||
63 | return infoBuilder.getInfo() | ||
64 | }) | ||
65 | } | ||
66 | |||
49 | async getSubtitles (): Promise<YoutubeDLSubs> { | 67 | async getSubtitles (): Promise<YoutubeDLSubs> { |
50 | const cwd = CONFIG.STORAGE.TMP_DIR | 68 | const cwd = CONFIG.STORAGE.TMP_DIR |
51 | 69 | ||
@@ -103,7 +121,7 @@ class YoutubeDLWrapper { | |||
103 | 121 | ||
104 | return remove(path) | 122 | return remove(path) |
105 | }) | 123 | }) |
106 | .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) | 124 | .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() })) |
107 | 125 | ||
108 | throw err | 126 | throw err |
109 | } | 127 | } |
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index f0f16d9bd..74c82541e 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts | |||
@@ -48,6 +48,7 @@ function checkConfig () { | |||
48 | checkRemoteRedundancyConfig() | 48 | checkRemoteRedundancyConfig() |
49 | checkStorageConfig() | 49 | checkStorageConfig() |
50 | checkTranscodingConfig() | 50 | checkTranscodingConfig() |
51 | checkImportConfig() | ||
51 | checkBroadcastMessageConfig() | 52 | checkBroadcastMessageConfig() |
52 | checkSearchConfig() | 53 | checkSearchConfig() |
53 | checkLiveConfig() | 54 | checkLiveConfig() |
@@ -200,6 +201,12 @@ function checkTranscodingConfig () { | |||
200 | } | 201 | } |
201 | } | 202 | } |
202 | 203 | ||
204 | function checkImportConfig () { | ||
205 | if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) { | ||
206 | throw new Error('You need to enable HTTP import to allow synchronization') | ||
207 | } | ||
208 | } | ||
209 | |||
203 | function checkBroadcastMessageConfig () { | 210 | function checkBroadcastMessageConfig () { |
204 | if (CONFIG.BROADCAST_MESSAGE.ENABLED) { | 211 | if (CONFIG.BROADCAST_MESSAGE.ENABLED) { |
205 | const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL | 212 | const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL |
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index f4057b81b..3188903be 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -32,6 +32,8 @@ function checkMissedConfig () { | |||
32 | 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', | 32 | 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', |
33 | 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled', | 33 | 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled', |
34 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', | 34 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', |
35 | 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', | ||
36 | 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', | ||
35 | 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', | 37 | 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', |
36 | 'client.videos.miniature.display_author_avatar', | 38 | 'client.videos.miniature.display_author_avatar', |
37 | 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', | 39 | 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 1a0b8942c..2c92bea22 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -398,6 +398,14 @@ const CONFIG = { | |||
398 | TORRENT: { | 398 | TORRENT: { |
399 | get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') } | 399 | get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') } |
400 | } | 400 | } |
401 | }, | ||
402 | VIDEO_CHANNEL_SYNCHRONIZATION: { | ||
403 | get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') }, | ||
404 | get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') }, | ||
405 | get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) }, | ||
406 | get VIDEOS_LIMIT_PER_SYNCHRONIZATION () { | ||
407 | return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization') | ||
408 | } | ||
401 | } | 409 | } |
402 | }, | 410 | }, |
403 | AUTO_BLACKLIST: { | 411 | AUTO_BLACKLIST: { |
@@ -499,6 +507,7 @@ const CONFIG = { | |||
499 | get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') } | 507 | get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') } |
500 | } | 508 | } |
501 | } | 509 | } |
510 | |||
502 | } | 511 | } |
503 | 512 | ||
504 | function registerConfigChangedHandler (fun: Function) { | 513 | function registerConfigChangedHandler (fun: Function) { |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5a5f2d666..697a64d42 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils' | |||
6 | import { | 6 | import { |
7 | AbuseState, | 7 | AbuseState, |
8 | JobType, | 8 | JobType, |
9 | VideoChannelSyncState, | ||
9 | VideoImportState, | 10 | VideoImportState, |
10 | VideoPrivacy, | 11 | VideoPrivacy, |
11 | VideoRateType, | 12 | VideoRateType, |
@@ -24,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' | |||
24 | 25 | ||
25 | // --------------------------------------------------------------------------- | 26 | // --------------------------------------------------------------------------- |
26 | 27 | ||
27 | const LAST_MIGRATION_VERSION = 725 | 28 | const LAST_MIGRATION_VERSION = 730 |
28 | 29 | ||
29 | // --------------------------------------------------------------------------- | 30 | // --------------------------------------------------------------------------- |
30 | 31 | ||
@@ -64,6 +65,7 @@ const SORTABLE_COLUMNS = { | |||
64 | JOBS: [ 'createdAt' ], | 65 | JOBS: [ 'createdAt' ], |
65 | VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], | 66 | VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], |
66 | VIDEO_IMPORTS: [ 'createdAt' ], | 67 | VIDEO_IMPORTS: [ 'createdAt' ], |
68 | VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ], | ||
67 | 69 | ||
68 | VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ], | 70 | VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ], |
69 | VIDEO_COMMENTS: [ 'createdAt' ], | 71 | VIDEO_COMMENTS: [ 'createdAt' ], |
@@ -156,6 +158,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { | |||
156 | 'video-live-ending': 1, | 158 | 'video-live-ending': 1, |
157 | 'video-studio-edition': 1, | 159 | 'video-studio-edition': 1, |
158 | 'manage-video-torrent': 1, | 160 | 'manage-video-torrent': 1, |
161 | 'video-channel-import': 1, | ||
162 | 'after-video-channel-import': 1, | ||
159 | 'move-to-object-storage': 3, | 163 | 'move-to-object-storage': 3, |
160 | 'notify': 1, | 164 | 'notify': 1, |
161 | 'federate-video': 1 | 165 | 'federate-video': 1 |
@@ -178,6 +182,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im | |||
178 | 'video-studio-edition': 1, | 182 | 'video-studio-edition': 1, |
179 | 'manage-video-torrent': 1, | 183 | 'manage-video-torrent': 1, |
180 | 'move-to-object-storage': 1, | 184 | 'move-to-object-storage': 1, |
185 | 'video-channel-import': 1, | ||
186 | 'after-video-channel-import': 1, | ||
181 | 'notify': 5, | 187 | 'notify': 5, |
182 | 'federate-video': 3 | 188 | 'federate-video': 3 |
183 | } | 189 | } |
@@ -199,9 +205,11 @@ const JOB_TTL: { [id in JobType]: number } = { | |||
199 | 'video-redundancy': 1000 * 3600 * 3, // 3 hours | 205 | 'video-redundancy': 1000 * 3600 * 3, // 3 hours |
200 | 'video-live-ending': 1000 * 60 * 10, // 10 minutes | 206 | 'video-live-ending': 1000 * 60 * 10, // 10 minutes |
201 | 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours | 207 | 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours |
208 | 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours | ||
209 | 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours | ||
210 | 'after-video-channel-import': 60000 * 5, // 5 minutes | ||
202 | 'notify': 60000 * 5, // 5 minutes | 211 | 'notify': 60000 * 5, // 5 minutes |
203 | 'federate-video': 60000 * 5, // 5 minutes | 212 | 'federate-video': 60000 * 5 // 5 minutes |
204 | 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours | ||
205 | } | 213 | } |
206 | const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = { | 214 | const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = { |
207 | 'videos-views-stats': { | 215 | 'videos-views-stats': { |
@@ -246,7 +254,8 @@ const SCHEDULER_INTERVALS_MS = { | |||
246 | REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day | 254 | REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day |
247 | REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day | 255 | REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day |
248 | UPDATE_INBOX_STATS: 1000 * 60, // 1 minute | 256 | UPDATE_INBOX_STATS: 1000 * 60, // 1 minute |
249 | REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour | 257 | REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour |
258 | CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL | ||
250 | } | 259 | } |
251 | 260 | ||
252 | // --------------------------------------------------------------------------- | 261 | // --------------------------------------------------------------------------- |
@@ -276,8 +285,12 @@ const CONSTRAINTS_FIELDS = { | |||
276 | NAME: { min: 1, max: 120 }, // Length | 285 | NAME: { min: 1, max: 120 }, // Length |
277 | DESCRIPTION: { min: 3, max: 1000 }, // Length | 286 | DESCRIPTION: { min: 3, max: 1000 }, // Length |
278 | SUPPORT: { min: 3, max: 1000 }, // Length | 287 | SUPPORT: { min: 3, max: 1000 }, // Length |
288 | EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length | ||
279 | URL: { min: 3, max: 2000 } // Length | 289 | URL: { min: 3, max: 2000 } // Length |
280 | }, | 290 | }, |
291 | VIDEO_CHANNEL_SYNCS: { | ||
292 | EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length | ||
293 | }, | ||
281 | VIDEO_CAPTIONS: { | 294 | VIDEO_CAPTIONS: { |
282 | CAPTION_FILE: { | 295 | CAPTION_FILE: { |
283 | EXTNAME: [ '.vtt', '.srt' ], | 296 | EXTNAME: [ '.vtt', '.srt' ], |
@@ -478,6 +491,13 @@ const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { | |||
478 | [VideoImportState.PROCESSING]: 'Processing' | 491 | [VideoImportState.PROCESSING]: 'Processing' |
479 | } | 492 | } |
480 | 493 | ||
494 | const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = { | ||
495 | [VideoChannelSyncState.FAILED]: 'Failed', | ||
496 | [VideoChannelSyncState.SYNCED]: 'Synchronized', | ||
497 | [VideoChannelSyncState.PROCESSING]: 'Processing', | ||
498 | [VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run' | ||
499 | } | ||
500 | |||
481 | const ABUSE_STATES: { [ id in AbuseState ]: string } = { | 501 | const ABUSE_STATES: { [ id in AbuseState ]: string } = { |
482 | [AbuseState.PENDING]: 'Pending', | 502 | [AbuseState.PENDING]: 'Pending', |
483 | [AbuseState.REJECTED]: 'Rejected', | 503 | [AbuseState.REJECTED]: 'Rejected', |
@@ -1005,6 +1025,7 @@ export { | |||
1005 | JOB_COMPLETED_LIFETIME, | 1025 | JOB_COMPLETED_LIFETIME, |
1006 | HTTP_SIGNATURE, | 1026 | HTTP_SIGNATURE, |
1007 | VIDEO_IMPORT_STATES, | 1027 | VIDEO_IMPORT_STATES, |
1028 | VIDEO_CHANNEL_SYNC_STATE, | ||
1008 | VIEW_LIFETIME, | 1029 | VIEW_LIFETIME, |
1009 | CONTACT_FORM_LIFETIME, | 1030 | CONTACT_FORM_LIFETIME, |
1010 | VIDEO_PLAYLIST_PRIVACIES, | 1031 | VIDEO_PLAYLIST_PRIVACIES, |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 91286241b..f55f40df0 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -50,6 +50,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla | |||
50 | import { VideoTagModel } from '../models/video/video-tag' | 50 | import { VideoTagModel } from '../models/video/video-tag' |
51 | import { VideoViewModel } from '../models/view/video-view' | 51 | import { VideoViewModel } from '../models/view/video-view' |
52 | import { CONFIG } from './config' | 52 | import { CONFIG } from './config' |
53 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
53 | 54 | ||
54 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 55 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
55 | 56 | ||
@@ -153,7 +154,8 @@ async function initDatabaseModels (silent: boolean) { | |||
153 | VideoTrackerModel, | 154 | VideoTrackerModel, |
154 | PluginModel, | 155 | PluginModel, |
155 | ActorCustomPageModel, | 156 | ActorCustomPageModel, |
156 | VideoJobInfoModel | 157 | VideoJobInfoModel, |
158 | VideoChannelSyncModel | ||
157 | ]) | 159 | ]) |
158 | 160 | ||
159 | // Check extensions exist in the database | 161 | // Check extensions exist in the database |
diff --git a/server/initializers/migrations/0730-video-channel-sync.ts b/server/initializers/migrations/0730-video-channel-sync.ts new file mode 100644 index 000000000..a2fe8211f --- /dev/null +++ b/server/initializers/migrations/0730-video-channel-sync.ts | |||
@@ -0,0 +1,36 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | async function up (utils: { | ||
4 | transaction: Sequelize.Transaction | ||
5 | queryInterface: Sequelize.QueryInterface | ||
6 | sequelize: Sequelize.Sequelize | ||
7 | db: any | ||
8 | }): Promise<void> { | ||
9 | const query = ` | ||
10 | CREATE TABLE IF NOT EXISTS "videoChannelSync" ( | ||
11 | "id" SERIAL, | ||
12 | "externalChannelUrl" VARCHAR(2000) NOT NULL DEFAULT NULL, | ||
13 | "videoChannelId" INTEGER NOT NULL REFERENCES "videoChannel" ("id") | ||
14 | ON DELETE CASCADE | ||
15 | ON UPDATE CASCADE, | ||
16 | "state" INTEGER NOT NULL DEFAULT 1, | ||
17 | "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, | ||
18 | "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, | ||
19 | "lastSyncAt" TIMESTAMP WITH TIME ZONE, | ||
20 | PRIMARY KEY ("id") | ||
21 | ); | ||
22 | ` | ||
23 | await utils.sequelize.query(query, { transaction: utils.transaction }) | ||
24 | } | ||
25 | |||
26 | async function down (utils: { | ||
27 | queryInterface: Sequelize.QueryInterface | ||
28 | transaction: Sequelize.Transaction | ||
29 | }) { | ||
30 | await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction }) | ||
31 | } | ||
32 | |||
33 | export { | ||
34 | up, | ||
35 | down | ||
36 | } | ||
diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts new file mode 100644 index 000000000..ffdd8c5b5 --- /dev/null +++ b/server/lib/job-queue/handlers/after-video-channel-import.ts | |||
@@ -0,0 +1,37 @@ | |||
1 | import { Job } from 'bullmq' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
4 | import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models' | ||
5 | |||
6 | export async function processAfterVideoChannelImport (job: Job) { | ||
7 | const payload = job.data as AfterVideoChannelImportPayload | ||
8 | if (!payload.channelSyncId) return | ||
9 | |||
10 | logger.info('Processing after video channel import in job %s.', job.id) | ||
11 | |||
12 | const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId) | ||
13 | if (!sync) { | ||
14 | logger.error('Unknown sync id %d.', payload.channelSyncId) | ||
15 | return | ||
16 | } | ||
17 | |||
18 | const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>() | ||
19 | |||
20 | let errors = 0 | ||
21 | let successes = 0 | ||
22 | |||
23 | for (const value of Object.values(childrenValues)) { | ||
24 | if (value.resultType === 'success') successes++ | ||
25 | else if (value.resultType === 'error') errors++ | ||
26 | } | ||
27 | |||
28 | if (errors > 0) { | ||
29 | sync.state = VideoChannelSyncState.FAILED | ||
30 | logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes }) | ||
31 | } else { | ||
32 | sync.state = VideoChannelSyncState.SYNCED | ||
33 | logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes }) | ||
34 | } | ||
35 | |||
36 | await sync.save() | ||
37 | } | ||
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts new file mode 100644 index 000000000..9bdb2d269 --- /dev/null +++ b/server/lib/job-queue/handlers/video-channel-import.ts | |||
@@ -0,0 +1,36 @@ | |||
1 | import { Job } from 'bullmq' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { synchronizeChannel } from '@server/lib/sync-channel' | ||
5 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
6 | import { VideoChannelImportPayload } from '@shared/models' | ||
7 | |||
8 | export async function processVideoChannelImport (job: Job) { | ||
9 | const payload = job.data as VideoChannelImportPayload | ||
10 | |||
11 | logger.info('Processing video channel import in job %s.', job.id) | ||
12 | |||
13 | // Channel import requires only http upload to be allowed | ||
14 | if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { | ||
15 | logger.error('Cannot import channel as the HTTP upload is disabled') | ||
16 | return | ||
17 | } | ||
18 | |||
19 | if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { | ||
20 | logger.error('Cannot import channel as the synchronization is disabled') | ||
21 | return | ||
22 | } | ||
23 | |||
24 | const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) | ||
25 | |||
26 | try { | ||
27 | logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) | ||
28 | |||
29 | await synchronizeChannel({ | ||
30 | channel: videoChannel, | ||
31 | externalChannelUrl: payload.externalChannelUrl | ||
32 | }) | ||
33 | } catch (err) { | ||
34 | logger.error(`Failed to import channel ${videoChannel.name}`, { err }) | ||
35 | } | ||
36 | } | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index f4629159c..9901b878c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths' | |||
8 | import { Hooks } from '@server/lib/plugins/hooks' | 8 | import { Hooks } from '@server/lib/plugins/hooks' |
9 | import { ServerConfigManager } from '@server/lib/server-config-manager' | 9 | import { ServerConfigManager } from '@server/lib/server-config-manager' |
10 | import { isAbleToUploadVideo } from '@server/lib/user' | 10 | import { isAbleToUploadVideo } from '@server/lib/user' |
11 | import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video' | 11 | import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video' |
12 | import { VideoPathManager } from '@server/lib/video-path-manager' | 12 | import { VideoPathManager } from '@server/lib/video-path-manager' |
13 | import { buildNextVideoState } from '@server/lib/video-state' | 13 | import { buildNextVideoState } from '@server/lib/video-state' |
14 | import { ThumbnailModel } from '@server/models/video/thumbnail' | 14 | import { ThumbnailModel } from '@server/models/video/thumbnail' |
@@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils' | |||
18 | import { | 18 | import { |
19 | ThumbnailType, | 19 | ThumbnailType, |
20 | VideoImportPayload, | 20 | VideoImportPayload, |
21 | VideoImportPreventExceptionResult, | ||
21 | VideoImportState, | 22 | VideoImportState, |
22 | VideoImportTorrentPayload, | 23 | VideoImportTorrentPayload, |
23 | VideoImportTorrentPayloadType, | 24 | VideoImportTorrentPayloadType, |
@@ -41,20 +42,29 @@ import { Notifier } from '../../notifier' | |||
41 | import { generateVideoMiniature } from '../../thumbnail' | 42 | import { generateVideoMiniature } from '../../thumbnail' |
42 | import { JobQueue } from '../job-queue' | 43 | import { JobQueue } from '../job-queue' |
43 | 44 | ||
44 | async function processVideoImport (job: Job) { | 45 | async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> { |
45 | const payload = job.data as VideoImportPayload | 46 | const payload = job.data as VideoImportPayload |
46 | 47 | ||
47 | const videoImport = await getVideoImportOrDie(payload) | 48 | const videoImport = await getVideoImportOrDie(payload) |
48 | if (videoImport.state === VideoImportState.CANCELLED) { | 49 | if (videoImport.state === VideoImportState.CANCELLED) { |
49 | logger.info('Do not process import since it has been cancelled', { payload }) | 50 | logger.info('Do not process import since it has been cancelled', { payload }) |
50 | return | 51 | return { resultType: 'success' } |
51 | } | 52 | } |
52 | 53 | ||
53 | videoImport.state = VideoImportState.PROCESSING | 54 | videoImport.state = VideoImportState.PROCESSING |
54 | await videoImport.save() | 55 | await videoImport.save() |
55 | 56 | ||
56 | if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload) | 57 | try { |
57 | if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload) | 58 | if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload) |
59 | if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload) | ||
60 | |||
61 | return { resultType: 'success' } | ||
62 | } catch (err) { | ||
63 | if (!payload.preventException) throw err | ||
64 | |||
65 | logger.warn('Catch error in video import to send value to parent job.', { payload, err }) | ||
66 | return { resultType: 'error' } | ||
67 | } | ||
58 | } | 68 | } |
59 | 69 | ||
60 | // --------------------------------------------------------------------------- | 70 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 281e2e51a..3970d48b7 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -22,6 +22,7 @@ import { | |||
22 | ActivitypubHttpFetcherPayload, | 22 | ActivitypubHttpFetcherPayload, |
23 | ActivitypubHttpUnicastPayload, | 23 | ActivitypubHttpUnicastPayload, |
24 | ActorKeysPayload, | 24 | ActorKeysPayload, |
25 | AfterVideoChannelImportPayload, | ||
25 | DeleteResumableUploadMetaFilePayload, | 26 | DeleteResumableUploadMetaFilePayload, |
26 | EmailPayload, | 27 | EmailPayload, |
27 | FederateVideoPayload, | 28 | FederateVideoPayload, |
@@ -31,6 +32,7 @@ import { | |||
31 | MoveObjectStoragePayload, | 32 | MoveObjectStoragePayload, |
32 | NotifyPayload, | 33 | NotifyPayload, |
33 | RefreshPayload, | 34 | RefreshPayload, |
35 | VideoChannelImportPayload, | ||
34 | VideoFileImportPayload, | 36 | VideoFileImportPayload, |
35 | VideoImportPayload, | 37 | VideoImportPayload, |
36 | VideoLiveEndingPayload, | 38 | VideoLiveEndingPayload, |
@@ -53,12 +55,14 @@ import { processFederateVideo } from './handlers/federate-video' | |||
53 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' | 55 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' |
54 | import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' | 56 | import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' |
55 | import { processNotify } from './handlers/notify' | 57 | import { processNotify } from './handlers/notify' |
58 | import { processVideoChannelImport } from './handlers/video-channel-import' | ||
56 | import { processVideoFileImport } from './handlers/video-file-import' | 59 | import { processVideoFileImport } from './handlers/video-file-import' |
57 | import { processVideoImport } from './handlers/video-import' | 60 | import { processVideoImport } from './handlers/video-import' |
58 | import { processVideoLiveEnding } from './handlers/video-live-ending' | 61 | import { processVideoLiveEnding } from './handlers/video-live-ending' |
59 | import { processVideoStudioEdition } from './handlers/video-studio-edition' | 62 | import { processVideoStudioEdition } from './handlers/video-studio-edition' |
60 | import { processVideoTranscoding } from './handlers/video-transcoding' | 63 | import { processVideoTranscoding } from './handlers/video-transcoding' |
61 | import { processVideosViewsStats } from './handlers/video-views-stats' | 64 | import { processVideosViewsStats } from './handlers/video-views-stats' |
65 | import { processAfterVideoChannelImport } from './handlers/after-video-channel-import' | ||
62 | 66 | ||
63 | export type CreateJobArgument = | 67 | export type CreateJobArgument = |
64 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 68 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -79,6 +83,9 @@ export type CreateJobArgument = | |||
79 | { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | | 83 | { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | |
80 | { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | | 84 | { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | |
81 | { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | | 85 | { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | |
86 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | | ||
87 | { type: 'video-channel-import', payload: VideoChannelImportPayload } | | ||
88 | { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | | ||
82 | { type: 'notify', payload: NotifyPayload } | | 89 | { type: 'notify', payload: NotifyPayload } | |
83 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | | 90 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | |
84 | { type: 'federate-video', payload: FederateVideoPayload } | 91 | { type: 'federate-video', payload: FederateVideoPayload } |
@@ -106,8 +113,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = { | |||
106 | 'video-redundancy': processVideoRedundancy, | 113 | 'video-redundancy': processVideoRedundancy, |
107 | 'move-to-object-storage': processMoveToObjectStorage, | 114 | 'move-to-object-storage': processMoveToObjectStorage, |
108 | 'manage-video-torrent': processManageVideoTorrent, | 115 | 'manage-video-torrent': processManageVideoTorrent, |
109 | 'notify': processNotify, | ||
110 | 'video-studio-edition': processVideoStudioEdition, | 116 | 'video-studio-edition': processVideoStudioEdition, |
117 | 'video-channel-import': processVideoChannelImport, | ||
118 | 'after-video-channel-import': processAfterVideoChannelImport, | ||
119 | 'notify': processNotify, | ||
111 | 'federate-video': processFederateVideo | 120 | 'federate-video': processFederateVideo |
112 | } | 121 | } |
113 | 122 | ||
@@ -134,6 +143,8 @@ const jobTypes: JobType[] = [ | |||
134 | 'move-to-object-storage', | 143 | 'move-to-object-storage', |
135 | 'manage-video-torrent', | 144 | 'manage-video-torrent', |
136 | 'video-studio-edition', | 145 | 'video-studio-edition', |
146 | 'video-channel-import', | ||
147 | 'after-video-channel-import', | ||
137 | 'notify', | 148 | 'notify', |
138 | 'federate-video' | 149 | 'federate-video' |
139 | ] | 150 | ] |
@@ -306,7 +317,7 @@ class JobQueue { | |||
306 | .catch(err => logger.error('Cannot create job.', { err, options })) | 317 | .catch(err => logger.error('Cannot create job.', { err, options })) |
307 | } | 318 | } |
308 | 319 | ||
309 | async createJob (options: CreateJobArgument & CreateJobOptions) { | 320 | createJob (options: CreateJobArgument & CreateJobOptions) { |
310 | const queue: Queue = this.queues[options.type] | 321 | const queue: Queue = this.queues[options.type] |
311 | if (queue === undefined) { | 322 | if (queue === undefined) { |
312 | logger.error('Unknown queue %s: cannot create job.', options.type) | 323 | logger.error('Unknown queue %s: cannot create job.', options.type) |
@@ -318,7 +329,7 @@ class JobQueue { | |||
318 | return queue.add('job', options.payload, jobOptions) | 329 | return queue.add('job', options.payload, jobOptions) |
319 | } | 330 | } |
320 | 331 | ||
321 | async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { | 332 | createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { |
322 | let lastJob: FlowJob | 333 | let lastJob: FlowJob |
323 | 334 | ||
324 | for (const job of jobs) { | 335 | for (const job of jobs) { |
@@ -336,7 +347,7 @@ class JobQueue { | |||
336 | return this.flowProducer.add(lastJob) | 347 | return this.flowProducer.add(lastJob) |
337 | } | 348 | } |
338 | 349 | ||
339 | async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { | 350 | createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { |
340 | return this.flowProducer.add({ | 351 | return this.flowProducer.add({ |
341 | ...this.buildJobFlowOption(parent), | 352 | ...this.buildJobFlowOption(parent), |
342 | 353 | ||
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts new file mode 100644 index 000000000..fd9a35299 --- /dev/null +++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts | |||
@@ -0,0 +1,61 @@ | |||
1 | import { logger } from '@server/helpers/logger' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
4 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
5 | import { VideoChannelSyncState } from '@shared/models' | ||
6 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
7 | import { synchronizeChannel } from '../sync-channel' | ||
8 | import { AbstractScheduler } from './abstract-scheduler' | ||
9 | |||
10 | export class VideoChannelSyncLatestScheduler extends AbstractScheduler { | ||
11 | private static instance: AbstractScheduler | ||
12 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL | ||
13 | |||
14 | private constructor () { | ||
15 | super() | ||
16 | } | ||
17 | |||
18 | protected async internalExecute () { | ||
19 | logger.debug('Running %s.%s', this.constructor.name, this.internalExecute.name) | ||
20 | |||
21 | if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { | ||
22 | logger.info('Discard channels synchronization as the feature is disabled') | ||
23 | return | ||
24 | } | ||
25 | |||
26 | const channelSyncs = await VideoChannelSyncModel.listSyncs() | ||
27 | |||
28 | for (const sync of channelSyncs) { | ||
29 | const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) | ||
30 | |||
31 | try { | ||
32 | logger.info( | ||
33 | 'Creating video import jobs for "%s" sync with external channel "%s"', | ||
34 | channel.Actor.preferredUsername, sync.externalChannelUrl | ||
35 | ) | ||
36 | |||
37 | const onlyAfter = sync.lastSyncAt || sync.createdAt | ||
38 | |||
39 | sync.state = VideoChannelSyncState.PROCESSING | ||
40 | sync.lastSyncAt = new Date() | ||
41 | await sync.save() | ||
42 | |||
43 | await synchronizeChannel({ | ||
44 | channel, | ||
45 | externalChannelUrl: sync.externalChannelUrl, | ||
46 | videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, | ||
47 | channelSync: sync, | ||
48 | onlyAfter | ||
49 | }) | ||
50 | } catch (err) { | ||
51 | logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err }) | ||
52 | sync.state = VideoChannelSyncState.FAILED | ||
53 | await sync.save() | ||
54 | } | ||
55 | } | ||
56 | } | ||
57 | |||
58 | static get Instance () { | ||
59 | return this.instance || (this.instance = new this()) | ||
60 | } | ||
61 | } | ||
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index a3312fa20..78a9546ae 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts | |||
@@ -170,6 +170,9 @@ class ServerConfigManager { | |||
170 | torrent: { | 170 | torrent: { |
171 | enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED | 171 | enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED |
172 | } | 172 | } |
173 | }, | ||
174 | videoChannelSynchronization: { | ||
175 | enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED | ||
173 | } | 176 | } |
174 | }, | 177 | }, |
175 | autoBlacklist: { | 178 | autoBlacklist: { |
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts new file mode 100644 index 000000000..50f80e6f9 --- /dev/null +++ b/server/lib/sync-channel.ts | |||
@@ -0,0 +1,81 @@ | |||
1 | import { logger } from '@server/helpers/logger' | ||
2 | import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { buildYoutubeDLImport } from '@server/lib/video-import' | ||
5 | import { UserModel } from '@server/models/user/user' | ||
6 | import { VideoImportModel } from '@server/models/video/video-import' | ||
7 | import { MChannelAccountDefault, MChannelSync } from '@server/types/models' | ||
8 | import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' | ||
9 | import { CreateJobArgument, JobQueue } from './job-queue' | ||
10 | import { ServerConfigManager } from './server-config-manager' | ||
11 | |||
12 | export async function synchronizeChannel (options: { | ||
13 | channel: MChannelAccountDefault | ||
14 | externalChannelUrl: string | ||
15 | channelSync?: MChannelSync | ||
16 | videosCountLimit?: number | ||
17 | onlyAfter?: Date | ||
18 | }) { | ||
19 | const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options | ||
20 | |||
21 | const user = await UserModel.loadByChannelActorId(channel.actorId) | ||
22 | const youtubeDL = new YoutubeDLWrapper( | ||
23 | externalChannelUrl, | ||
24 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | ||
25 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
26 | ) | ||
27 | |||
28 | const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) | ||
29 | |||
30 | const targetUrls = infoList | ||
31 | .filter(videoInfo => { | ||
32 | if (!onlyAfter) return true | ||
33 | |||
34 | return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime() | ||
35 | }) | ||
36 | .map(videoInfo => videoInfo.webpageUrl) | ||
37 | |||
38 | logger.info( | ||
39 | 'Fetched %d candidate URLs for sync channel %s.', | ||
40 | targetUrls.length, channel.Actor.preferredUsername, { targetUrls } | ||
41 | ) | ||
42 | |||
43 | if (targetUrls.length === 0) { | ||
44 | if (channelSync) { | ||
45 | channelSync.state = VideoChannelSyncState.SYNCED | ||
46 | await channelSync.save() | ||
47 | } | ||
48 | |||
49 | return | ||
50 | } | ||
51 | |||
52 | const children: CreateJobArgument[] = [] | ||
53 | |||
54 | for (const targetUrl of targetUrls) { | ||
55 | if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { | ||
56 | logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl) | ||
57 | continue | ||
58 | } | ||
59 | |||
60 | const { job } = await buildYoutubeDLImport({ | ||
61 | user, | ||
62 | channel, | ||
63 | targetUrl, | ||
64 | channelSync, | ||
65 | importDataOverride: { | ||
66 | privacy: VideoPrivacy.PUBLIC | ||
67 | } | ||
68 | }) | ||
69 | |||
70 | children.push(job) | ||
71 | } | ||
72 | |||
73 | const parent: CreateJobArgument = { | ||
74 | type: 'after-video-channel-import', | ||
75 | payload: { | ||
76 | channelSyncId: channelSync?.id | ||
77 | } | ||
78 | } | ||
79 | |||
80 | await JobQueue.Instance.createJobWithChildren(parent, children) | ||
81 | } | ||
diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts new file mode 100644 index 000000000..fb9306967 --- /dev/null +++ b/server/lib/video-import.ts | |||
@@ -0,0 +1,308 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils' | ||
3 | import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' | ||
4 | import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' | ||
5 | import { isResolvingToUnicastOnly } from '@server/helpers/dns' | ||
6 | import { logger } from '@server/helpers/logger' | ||
7 | import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl' | ||
8 | import { CONFIG } from '@server/initializers/config' | ||
9 | import { sequelizeTypescript } from '@server/initializers/database' | ||
10 | import { Hooks } from '@server/lib/plugins/hooks' | ||
11 | import { ServerConfigManager } from '@server/lib/server-config-manager' | ||
12 | import { setVideoTags } from '@server/lib/video' | ||
13 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | ||
14 | import { VideoModel } from '@server/models/video/video' | ||
15 | import { VideoCaptionModel } from '@server/models/video/video-caption' | ||
16 | import { VideoImportModel } from '@server/models/video/video-import' | ||
17 | import { FilteredModelAttributes } from '@server/types' | ||
18 | import { | ||
19 | MChannelAccountDefault, | ||
20 | MChannelSync, | ||
21 | MThumbnail, | ||
22 | MUser, | ||
23 | MVideoAccountDefault, | ||
24 | MVideoCaption, | ||
25 | MVideoImportFormattable, | ||
26 | MVideoTag, | ||
27 | MVideoThumbnail, | ||
28 | MVideoWithBlacklistLight | ||
29 | } from '@server/types/models' | ||
30 | import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models' | ||
31 | import { getLocalVideoActivityPubUrl } from './activitypub/url' | ||
32 | import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail' | ||
33 | |||
34 | class YoutubeDlImportError extends Error { | ||
35 | code: YoutubeDlImportError.CODE | ||
36 | cause?: Error // Property to remove once ES2022 is used | ||
37 | constructor ({ message, code }) { | ||
38 | super(message) | ||
39 | this.code = code | ||
40 | } | ||
41 | |||
42 | static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) { | ||
43 | const ytDlErr = new this({ message: message ?? err.message, code }) | ||
44 | ytDlErr.cause = err | ||
45 | ytDlErr.stack = err.stack // Useless once ES2022 is used | ||
46 | return ytDlErr | ||
47 | } | ||
48 | } | ||
49 | |||
50 | namespace YoutubeDlImportError { | ||
51 | export enum CODE { | ||
52 | FETCH_ERROR, | ||
53 | NOT_ONLY_UNICAST_URL | ||
54 | } | ||
55 | } | ||
56 | |||
57 | // --------------------------------------------------------------------------- | ||
58 | |||
59 | async function insertFromImportIntoDB (parameters: { | ||
60 | video: MVideoThumbnail | ||
61 | thumbnailModel: MThumbnail | ||
62 | previewModel: MThumbnail | ||
63 | videoChannel: MChannelAccountDefault | ||
64 | tags: string[] | ||
65 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> | ||
66 | user: MUser | ||
67 | }): Promise<MVideoImportFormattable> { | ||
68 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters | ||
69 | |||
70 | const videoImport = await sequelizeTypescript.transaction(async t => { | ||
71 | const sequelizeOptions = { transaction: t } | ||
72 | |||
73 | // Save video object in database | ||
74 | const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) | ||
75 | videoCreated.VideoChannel = videoChannel | ||
76 | |||
77 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | ||
78 | if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) | ||
79 | |||
80 | await autoBlacklistVideoIfNeeded({ | ||
81 | video: videoCreated, | ||
82 | user, | ||
83 | notify: false, | ||
84 | isRemote: false, | ||
85 | isNew: true, | ||
86 | transaction: t | ||
87 | }) | ||
88 | |||
89 | await setVideoTags({ video: videoCreated, tags, transaction: t }) | ||
90 | |||
91 | // Create video import object in database | ||
92 | const videoImport = await VideoImportModel.create( | ||
93 | Object.assign({ videoId: videoCreated.id }, videoImportAttributes), | ||
94 | sequelizeOptions | ||
95 | ) as MVideoImportFormattable | ||
96 | videoImport.Video = videoCreated | ||
97 | |||
98 | return videoImport | ||
99 | }) | ||
100 | |||
101 | return videoImport | ||
102 | } | ||
103 | |||
104 | async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: { | ||
105 | channelId: number | ||
106 | importData: YoutubeDLInfo | ||
107 | importDataOverride?: Partial<VideoImportCreate> | ||
108 | importType: 'url' | 'torrent' | ||
109 | }): Promise<MVideoThumbnail> { | ||
110 | let videoData = { | ||
111 | name: importDataOverride?.name || importData.name || 'Unknown name', | ||
112 | remote: false, | ||
113 | category: importDataOverride?.category || importData.category, | ||
114 | licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, | ||
115 | language: importDataOverride?.language || importData.language, | ||
116 | commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, | ||
117 | downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, | ||
118 | waitTranscoding: importDataOverride?.waitTranscoding || false, | ||
119 | state: VideoState.TO_IMPORT, | ||
120 | nsfw: importDataOverride?.nsfw || importData.nsfw || false, | ||
121 | description: importDataOverride?.description || importData.description, | ||
122 | support: importDataOverride?.support || null, | ||
123 | privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE, | ||
124 | duration: 0, // duration will be set by the import job | ||
125 | channelId, | ||
126 | originallyPublishedAt: importDataOverride?.originallyPublishedAt | ||
127 | ? new Date(importDataOverride?.originallyPublishedAt) | ||
128 | : importData.originallyPublishedAt | ||
129 | } | ||
130 | |||
131 | videoData = await Hooks.wrapObject( | ||
132 | videoData, | ||
133 | importType === 'url' | ||
134 | ? 'filter:api.video.import-url.video-attribute.result' | ||
135 | : 'filter:api.video.import-torrent.video-attribute.result' | ||
136 | ) | ||
137 | |||
138 | const video = new VideoModel(videoData) | ||
139 | video.url = getLocalVideoActivityPubUrl(video) | ||
140 | |||
141 | return video | ||
142 | } | ||
143 | |||
144 | async function buildYoutubeDLImport (options: { | ||
145 | targetUrl: string | ||
146 | channel: MChannelAccountDefault | ||
147 | user: MUser | ||
148 | channelSync?: MChannelSync | ||
149 | importDataOverride?: Partial<VideoImportCreate> | ||
150 | thumbnailFilePath?: string | ||
151 | previewFilePath?: string | ||
152 | }) { | ||
153 | const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options | ||
154 | |||
155 | const youtubeDL = new YoutubeDLWrapper( | ||
156 | targetUrl, | ||
157 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | ||
158 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
159 | ) | ||
160 | |||
161 | // Get video infos | ||
162 | let youtubeDLInfo: YoutubeDLInfo | ||
163 | try { | ||
164 | youtubeDLInfo = await youtubeDL.getInfoForDownload() | ||
165 | } catch (err) { | ||
166 | throw YoutubeDlImportError.fromError( | ||
167 | err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}` | ||
168 | ) | ||
169 | } | ||
170 | |||
171 | if (!await hasUnicastURLsOnly(youtubeDLInfo)) { | ||
172 | throw new YoutubeDlImportError({ | ||
173 | message: 'Cannot use non unicast IP as targetUrl.', | ||
174 | code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL | ||
175 | }) | ||
176 | } | ||
177 | |||
178 | const video = await buildVideoFromImport({ | ||
179 | channelId: channel.id, | ||
180 | importData: youtubeDLInfo, | ||
181 | importDataOverride, | ||
182 | importType: 'url' | ||
183 | }) | ||
184 | |||
185 | const thumbnailModel = await forgeThumbnail({ | ||
186 | inputPath: thumbnailFilePath, | ||
187 | downloadUrl: youtubeDLInfo.thumbnailUrl, | ||
188 | video, | ||
189 | type: ThumbnailType.MINIATURE | ||
190 | }) | ||
191 | |||
192 | const previewModel = await forgeThumbnail({ | ||
193 | inputPath: previewFilePath, | ||
194 | downloadUrl: youtubeDLInfo.thumbnailUrl, | ||
195 | video, | ||
196 | type: ThumbnailType.PREVIEW | ||
197 | }) | ||
198 | |||
199 | const videoImport = await insertFromImportIntoDB({ | ||
200 | video, | ||
201 | thumbnailModel, | ||
202 | previewModel, | ||
203 | videoChannel: channel, | ||
204 | tags: importDataOverride?.tags || youtubeDLInfo.tags, | ||
205 | user, | ||
206 | videoImportAttributes: { | ||
207 | targetUrl, | ||
208 | state: VideoImportState.PENDING, | ||
209 | userId: user.id | ||
210 | } | ||
211 | }) | ||
212 | |||
213 | // Get video subtitles | ||
214 | await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) | ||
215 | |||
216 | let fileExt = `.${youtubeDLInfo.ext}` | ||
217 | if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' | ||
218 | |||
219 | const payload: VideoImportPayload = { | ||
220 | type: 'youtube-dl' as 'youtube-dl', | ||
221 | videoImportId: videoImport.id, | ||
222 | fileExt, | ||
223 | // If part of a sync process, there is a parent job that will aggregate children results | ||
224 | preventException: !!channelSync | ||
225 | } | ||
226 | |||
227 | return { | ||
228 | videoImport, | ||
229 | job: { type: 'video-import' as 'video-import', payload } | ||
230 | } | ||
231 | } | ||
232 | |||
233 | // --------------------------------------------------------------------------- | ||
234 | |||
235 | export { | ||
236 | buildYoutubeDLImport, | ||
237 | YoutubeDlImportError, | ||
238 | insertFromImportIntoDB, | ||
239 | buildVideoFromImport | ||
240 | } | ||
241 | |||
242 | // --------------------------------------------------------------------------- | ||
243 | |||
244 | async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: { | ||
245 | inputPath?: string | ||
246 | downloadUrl?: string | ||
247 | video: MVideoThumbnail | ||
248 | type: ThumbnailType | ||
249 | }): Promise<MThumbnail> { | ||
250 | if (inputPath) { | ||
251 | return updateVideoMiniatureFromExisting({ | ||
252 | inputPath, | ||
253 | video, | ||
254 | type, | ||
255 | automaticallyGenerated: false | ||
256 | }) | ||
257 | } else if (downloadUrl) { | ||
258 | try { | ||
259 | return await updateVideoMiniatureFromUrl({ downloadUrl, video, type }) | ||
260 | } catch (err) { | ||
261 | logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err }) | ||
262 | } | ||
263 | } | ||
264 | return null | ||
265 | } | ||
266 | |||
267 | async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { | ||
268 | try { | ||
269 | const subtitles = await youtubeDL.getSubtitles() | ||
270 | |||
271 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) | ||
272 | |||
273 | for (const subtitle of subtitles) { | ||
274 | if (!await isVTTFileValid(subtitle.path)) { | ||
275 | await remove(subtitle.path) | ||
276 | continue | ||
277 | } | ||
278 | |||
279 | const videoCaption = new VideoCaptionModel({ | ||
280 | videoId, | ||
281 | language: subtitle.language, | ||
282 | filename: VideoCaptionModel.generateCaptionName(subtitle.language) | ||
283 | }) as MVideoCaption | ||
284 | |||
285 | // Move physical file | ||
286 | await moveAndProcessCaptionFile(subtitle, videoCaption) | ||
287 | |||
288 | await sequelizeTypescript.transaction(async t => { | ||
289 | await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) | ||
290 | }) | ||
291 | } | ||
292 | } catch (err) { | ||
293 | logger.warn('Cannot get video subtitles.', { err }) | ||
294 | } | ||
295 | } | ||
296 | |||
297 | async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { | ||
298 | const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) | ||
299 | const uniqHosts = new Set(hosts) | ||
300 | |||
301 | for (const h of uniqHosts) { | ||
302 | if (await isResolvingToUnicastOnly(h) !== true) { | ||
303 | return false | ||
304 | } | ||
305 | } | ||
306 | |||
307 | return true | ||
308 | } | ||
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index 9ce47c5aa..f60103f48 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts | |||
@@ -66,6 +66,8 @@ const customConfigUpdateValidator = [ | |||
66 | body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'), | 66 | body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'), |
67 | body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'), | 67 | body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'), |
68 | 68 | ||
69 | body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'), | ||
70 | |||
69 | body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'), | 71 | body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'), |
70 | body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'), | 72 | body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'), |
71 | 73 | ||
@@ -110,6 +112,7 @@ const customConfigUpdateValidator = [ | |||
110 | if (areValidationErrors(req, res)) return | 112 | if (areValidationErrors(req, res)) return |
111 | if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return | 113 | if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return |
112 | if (!checkInvalidTranscodingConfig(req.body, res)) return | 114 | if (!checkInvalidTranscodingConfig(req.body, res)) return |
115 | if (!checkInvalidSynchronizationConfig(req.body, res)) return | ||
113 | if (!checkInvalidLiveConfig(req.body, res)) return | 116 | if (!checkInvalidLiveConfig(req.body, res)) return |
114 | if (!checkInvalidVideoStudioConfig(req.body, res)) return | 117 | if (!checkInvalidVideoStudioConfig(req.body, res)) return |
115 | 118 | ||
@@ -157,6 +160,14 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express | |||
157 | return true | 160 | return true |
158 | } | 161 | } |
159 | 162 | ||
163 | function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) { | ||
164 | if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) { | ||
165 | res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' }) | ||
166 | return false | ||
167 | } | ||
168 | return true | ||
169 | } | ||
170 | |||
160 | function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { | 171 | function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { |
161 | if (customConfig.live.enabled === false) return true | 172 | if (customConfig.live.enabled === false) return true |
162 | 173 | ||
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index c9978e3b4..0354e3fc6 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts | |||
@@ -52,6 +52,7 @@ const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAY | |||
52 | const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS) | 52 | const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS) |
53 | const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) | 53 | const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) |
54 | const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) | 54 | const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) |
55 | const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS) | ||
55 | 56 | ||
56 | const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS) | 57 | const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS) |
57 | const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS) | 58 | const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS) |
@@ -84,5 +85,6 @@ export { | |||
84 | videoPlaylistsSearchSortValidator, | 85 | videoPlaylistsSearchSortValidator, |
85 | accountsFollowersSortValidator, | 86 | accountsFollowersSortValidator, |
86 | videoChannelsFollowersSortValidator, | 87 | videoChannelsFollowersSortValidator, |
88 | videoChannelSyncsSortValidator, | ||
87 | pluginsSortValidator | 89 | pluginsSortValidator |
88 | } | 90 | } |
diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts index 1dd7b5d2e..d225dfe45 100644 --- a/server/middlewares/validators/videos/index.ts +++ b/server/middlewares/validators/videos/index.ts | |||
@@ -14,3 +14,4 @@ export * from './video-stats' | |||
14 | export * from './video-studio' | 14 | export * from './video-studio' |
15 | export * from './video-transcoding' | 15 | export * from './video-transcoding' |
16 | export * from './videos' | 16 | export * from './videos' |
17 | export * from './video-channel-sync' | ||
diff --git a/server/middlewares/validators/videos/video-channel-sync.ts b/server/middlewares/validators/videos/video-channel-sync.ts new file mode 100644 index 000000000..b18498243 --- /dev/null +++ b/server/middlewares/validators/videos/video-channel-sync.ts | |||
@@ -0,0 +1,66 @@ | |||
1 | import * as express from 'express' | ||
2 | import { body, param } from 'express-validator' | ||
3 | import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { CONFIG } from '@server/initializers/config' | ||
6 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
7 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
8 | import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models' | ||
9 | import { areValidationErrors, doesVideoChannelIdExist } from '../shared' | ||
10 | |||
11 | export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
12 | if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { | ||
13 | return res.fail({ | ||
14 | status: HttpStatusCode.FORBIDDEN_403, | ||
15 | message: 'Synchronization is impossible as video channel synchronization is not enabled on the server' | ||
16 | }) | ||
17 | } | ||
18 | |||
19 | return next() | ||
20 | } | ||
21 | |||
22 | export const videoChannelSyncValidator = [ | ||
23 | body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'), | ||
24 | body('videoChannelId').isInt().withMessage('Should have a valid video channel id'), | ||
25 | |||
26 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
27 | logger.debug('Checking videoChannelSync parameters', { parameters: req.body }) | ||
28 | |||
29 | if (areValidationErrors(req, res)) return | ||
30 | |||
31 | const body: VideoChannelSyncCreate = req.body | ||
32 | if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return | ||
33 | |||
34 | const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId) | ||
35 | if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) { | ||
36 | return res.fail({ | ||
37 | message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations` | ||
38 | }) | ||
39 | } | ||
40 | |||
41 | return next() | ||
42 | } | ||
43 | ] | ||
44 | |||
45 | export const ensureSyncExists = [ | ||
46 | param('id').exists().isInt().withMessage('Should have an sync id'), | ||
47 | |||
48 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
49 | if (areValidationErrors(req, res)) return | ||
50 | |||
51 | const syncId = parseInt(req.params.id, 10) | ||
52 | const sync = await VideoChannelSyncModel.loadWithChannel(syncId) | ||
53 | |||
54 | if (!sync) { | ||
55 | return res.fail({ | ||
56 | status: HttpStatusCode.NOT_FOUND_404, | ||
57 | message: 'Synchronization not found' | ||
58 | }) | ||
59 | } | ||
60 | |||
61 | res.locals.videoChannelSync = sync | ||
62 | res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) | ||
63 | |||
64 | return next() | ||
65 | } | ||
66 | ] | ||
diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts index 3bfdebbb1..88f8b814d 100644 --- a/server/middlewares/validators/videos/video-channels.ts +++ b/server/middlewares/validators/videos/video-channels.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { body, param, query } from 'express-validator' | 2 | import { body, param, query } from 'express-validator' |
3 | import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' | ||
3 | import { CONFIG } from '@server/initializers/config' | 4 | import { CONFIG } from '@server/initializers/config' |
4 | import { MChannelAccountDefault } from '@server/types/models' | 5 | import { MChannelAccountDefault } from '@server/types/models' |
5 | import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' | 6 | import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' |
@@ -13,9 +14,9 @@ import { | |||
13 | import { logger } from '../../../helpers/logger' | 14 | import { logger } from '../../../helpers/logger' |
14 | import { ActorModel } from '../../../models/actor/actor' | 15 | import { ActorModel } from '../../../models/actor/actor' |
15 | import { VideoChannelModel } from '../../../models/video/video-channel' | 16 | import { VideoChannelModel } from '../../../models/video/video-channel' |
16 | import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared' | 17 | import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared' |
17 | 18 | ||
18 | const videoChannelsAddValidator = [ | 19 | export const videoChannelsAddValidator = [ |
19 | body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'), | 20 | body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'), |
20 | body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'), | 21 | body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'), |
21 | body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'), | 22 | body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'), |
@@ -45,7 +46,7 @@ const videoChannelsAddValidator = [ | |||
45 | } | 46 | } |
46 | ] | 47 | ] |
47 | 48 | ||
48 | const videoChannelsUpdateValidator = [ | 49 | export const videoChannelsUpdateValidator = [ |
49 | param('nameWithHost').exists().withMessage('Should have an video channel name with host'), | 50 | param('nameWithHost').exists().withMessage('Should have an video channel name with host'), |
50 | body('displayName') | 51 | body('displayName') |
51 | .optional() | 52 | .optional() |
@@ -69,7 +70,7 @@ const videoChannelsUpdateValidator = [ | |||
69 | } | 70 | } |
70 | ] | 71 | ] |
71 | 72 | ||
72 | const videoChannelsRemoveValidator = [ | 73 | export const videoChannelsRemoveValidator = [ |
73 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 74 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
74 | logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params }) | 75 | logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params }) |
75 | 76 | ||
@@ -79,7 +80,7 @@ const videoChannelsRemoveValidator = [ | |||
79 | } | 80 | } |
80 | ] | 81 | ] |
81 | 82 | ||
82 | const videoChannelsNameWithHostValidator = [ | 83 | export const videoChannelsNameWithHostValidator = [ |
83 | param('nameWithHost').exists().withMessage('Should have an video channel name with host'), | 84 | param('nameWithHost').exists().withMessage('Should have an video channel name with host'), |
84 | 85 | ||
85 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 86 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
@@ -93,7 +94,7 @@ const videoChannelsNameWithHostValidator = [ | |||
93 | } | 94 | } |
94 | ] | 95 | ] |
95 | 96 | ||
96 | const ensureIsLocalChannel = [ | 97 | export const ensureIsLocalChannel = [ |
97 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | 98 | (req: express.Request, res: express.Response, next: express.NextFunction) => { |
98 | if (res.locals.videoChannel.Actor.isOwned() === false) { | 99 | if (res.locals.videoChannel.Actor.isOwned() === false) { |
99 | return res.fail({ | 100 | return res.fail({ |
@@ -106,7 +107,18 @@ const ensureIsLocalChannel = [ | |||
106 | } | 107 | } |
107 | ] | 108 | ] |
108 | 109 | ||
109 | const videoChannelStatsValidator = [ | 110 | export const ensureChannelOwnerCanUpload = [ |
111 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
112 | const channel = res.locals.videoChannel | ||
113 | const user = { id: channel.Account.userId } | ||
114 | |||
115 | if (!await checkUserQuota(user, 1, res)) return | ||
116 | |||
117 | next() | ||
118 | } | ||
119 | ] | ||
120 | |||
121 | export const videoChannelStatsValidator = [ | ||
110 | query('withStats') | 122 | query('withStats') |
111 | .optional() | 123 | .optional() |
112 | .customSanitizer(toBooleanOrNull) | 124 | .customSanitizer(toBooleanOrNull) |
@@ -118,7 +130,7 @@ const videoChannelStatsValidator = [ | |||
118 | } | 130 | } |
119 | ] | 131 | ] |
120 | 132 | ||
121 | const videoChannelsListValidator = [ | 133 | export const videoChannelsListValidator = [ |
122 | query('search').optional().not().isEmpty().withMessage('Should have a valid search'), | 134 | query('search').optional().not().isEmpty().withMessage('Should have a valid search'), |
123 | 135 | ||
124 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | 136 | (req: express.Request, res: express.Response, next: express.NextFunction) => { |
@@ -130,17 +142,24 @@ const videoChannelsListValidator = [ | |||
130 | } | 142 | } |
131 | ] | 143 | ] |
132 | 144 | ||
133 | // --------------------------------------------------------------------------- | 145 | export const videoChannelImportVideosValidator = [ |
146 | body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'), | ||
134 | 147 | ||
135 | export { | 148 | (req: express.Request, res: express.Response, next: express.NextFunction) => { |
136 | videoChannelsAddValidator, | 149 | logger.debug('Checking videoChannelImport parameters', { parameters: req.body }) |
137 | videoChannelsUpdateValidator, | 150 | |
138 | videoChannelsRemoveValidator, | 151 | if (areValidationErrors(req, res)) return |
139 | videoChannelsNameWithHostValidator, | 152 | |
140 | ensureIsLocalChannel, | 153 | if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { |
141 | videoChannelsListValidator, | 154 | return res.fail({ |
142 | videoChannelStatsValidator | 155 | status: HttpStatusCode.FORBIDDEN_403, |
143 | } | 156 | message: 'Channel import is impossible as video upload via HTTP is not enabled on the server' |
157 | }) | ||
158 | } | ||
159 | |||
160 | return next() | ||
161 | } | ||
162 | ] | ||
144 | 163 | ||
145 | // --------------------------------------------------------------------------- | 164 | // --------------------------------------------------------------------------- |
146 | 165 | ||
diff --git a/server/models/utils.ts b/server/models/utils.ts index c468f748d..1e168d419 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts | |||
@@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A | |||
117 | return getSort(value, lastSort) | 117 | return getSort(value, lastSort) |
118 | } | 118 | } |
119 | 119 | ||
120 | function getChannelSyncSort (value: string): OrderItem[] { | ||
121 | const { direction, field } = buildDirectionAndField(value) | ||
122 | if (field.toLowerCase() === 'videochannel') { | ||
123 | return [ | ||
124 | [ literal('"VideoChannel.name"'), direction ] | ||
125 | ] | ||
126 | } | ||
127 | return [ [ field, direction ] ] | ||
128 | } | ||
129 | |||
120 | function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { | 130 | function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { |
121 | if (!model.createdAt || !model.updatedAt) { | 131 | if (!model.createdAt || !model.updatedAt) { |
122 | throw new Error('Miss createdAt & updatedAt attributes to model') | 132 | throw new Error('Miss createdAt & updatedAt attributes to model') |
@@ -280,6 +290,7 @@ export { | |||
280 | getAdminUsersSort, | 290 | getAdminUsersSort, |
281 | getVideoSort, | 291 | getVideoSort, |
282 | getBlacklistSort, | 292 | getBlacklistSort, |
293 | getChannelSyncSort, | ||
283 | createSimilarityAttribute, | 294 | createSimilarityAttribute, |
284 | throwIfNotValid, | 295 | throwIfNotValid, |
285 | buildServerIdsFollowedBy, | 296 | buildServerIdsFollowedBy, |
diff --git a/server/models/video/video-channel-sync.ts b/server/models/video/video-channel-sync.ts new file mode 100644 index 000000000..6e49cde10 --- /dev/null +++ b/server/models/video/video-channel-sync.ts | |||
@@ -0,0 +1,176 @@ | |||
1 | import { Op } from 'sequelize' | ||
2 | import { | ||
3 | AllowNull, | ||
4 | BelongsTo, | ||
5 | Column, | ||
6 | CreatedAt, | ||
7 | DataType, | ||
8 | Default, | ||
9 | DefaultScope, | ||
10 | ForeignKey, | ||
11 | Is, | ||
12 | Model, | ||
13 | Table, | ||
14 | UpdatedAt | ||
15 | } from 'sequelize-typescript' | ||
16 | import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' | ||
17 | import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs' | ||
18 | import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants' | ||
19 | import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models' | ||
20 | import { VideoChannelSync, VideoChannelSyncState } from '@shared/models' | ||
21 | import { AttributesOnly } from '@shared/typescript-utils' | ||
22 | import { AccountModel } from '../account/account' | ||
23 | import { UserModel } from '../user/user' | ||
24 | import { getChannelSyncSort, throwIfNotValid } from '../utils' | ||
25 | import { VideoChannelModel } from './video-channel' | ||
26 | |||
27 | @DefaultScope(() => ({ | ||
28 | include: [ | ||
29 | { | ||
30 | model: VideoChannelModel, // Default scope includes avatar and server | ||
31 | required: true | ||
32 | } | ||
33 | ] | ||
34 | })) | ||
35 | @Table({ | ||
36 | tableName: 'videoChannelSync', | ||
37 | indexes: [ | ||
38 | { | ||
39 | fields: [ 'videoChannelId' ] | ||
40 | } | ||
41 | ] | ||
42 | }) | ||
43 | export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> { | ||
44 | |||
45 | @AllowNull(false) | ||
46 | @Default(null) | ||
47 | @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true)) | ||
48 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max)) | ||
49 | externalChannelUrl: string | ||
50 | |||
51 | @CreatedAt | ||
52 | createdAt: Date | ||
53 | |||
54 | @UpdatedAt | ||
55 | updatedAt: Date | ||
56 | |||
57 | @ForeignKey(() => VideoChannelModel) | ||
58 | @Column | ||
59 | videoChannelId: number | ||
60 | |||
61 | @BelongsTo(() => VideoChannelModel, { | ||
62 | foreignKey: { | ||
63 | allowNull: false | ||
64 | }, | ||
65 | onDelete: 'cascade' | ||
66 | }) | ||
67 | VideoChannel: VideoChannelModel | ||
68 | |||
69 | @AllowNull(false) | ||
70 | @Default(VideoChannelSyncState.WAITING_FIRST_RUN) | ||
71 | @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state')) | ||
72 | @Column | ||
73 | state: VideoChannelSyncState | ||
74 | |||
75 | @AllowNull(true) | ||
76 | @Column(DataType.DATE) | ||
77 | lastSyncAt: Date | ||
78 | |||
79 | static listByAccountForAPI (options: { | ||
80 | accountId: number | ||
81 | start: number | ||
82 | count: number | ||
83 | sort: string | ||
84 | }) { | ||
85 | const getQuery = (forCount: boolean) => { | ||
86 | const videoChannelModel = forCount | ||
87 | ? VideoChannelModel.unscoped() | ||
88 | : VideoChannelModel | ||
89 | |||
90 | return { | ||
91 | offset: options.start, | ||
92 | limit: options.count, | ||
93 | order: getChannelSyncSort(options.sort), | ||
94 | include: [ | ||
95 | { | ||
96 | model: videoChannelModel, | ||
97 | required: true, | ||
98 | where: { | ||
99 | accountId: options.accountId | ||
100 | } | ||
101 | } | ||
102 | ] | ||
103 | } | ||
104 | } | ||
105 | |||
106 | return Promise.all([ | ||
107 | VideoChannelSyncModel.unscoped().count(getQuery(true)), | ||
108 | VideoChannelSyncModel.unscoped().findAll(getQuery(false)) | ||
109 | ]).then(([ total, data ]) => ({ total, data })) | ||
110 | } | ||
111 | |||
112 | static countByAccount (accountId: number) { | ||
113 | const query = { | ||
114 | include: [ | ||
115 | { | ||
116 | model: VideoChannelModel.unscoped(), | ||
117 | required: true, | ||
118 | where: { | ||
119 | accountId | ||
120 | } | ||
121 | } | ||
122 | ] | ||
123 | } | ||
124 | |||
125 | return VideoChannelSyncModel.unscoped().count(query) | ||
126 | } | ||
127 | |||
128 | static loadWithChannel (id: number): Promise<MChannelSyncChannel> { | ||
129 | return VideoChannelSyncModel.findByPk(id) | ||
130 | } | ||
131 | |||
132 | static async listSyncs (): Promise<MChannelSync[]> { | ||
133 | const query = { | ||
134 | include: [ | ||
135 | { | ||
136 | model: VideoChannelModel.unscoped(), | ||
137 | required: true, | ||
138 | include: [ | ||
139 | { | ||
140 | model: AccountModel.unscoped(), | ||
141 | required: true, | ||
142 | include: [ { | ||
143 | attributes: [], | ||
144 | model: UserModel.unscoped(), | ||
145 | required: true, | ||
146 | where: { | ||
147 | videoQuota: { | ||
148 | [Op.ne]: 0 | ||
149 | }, | ||
150 | videoQuotaDaily: { | ||
151 | [Op.ne]: 0 | ||
152 | } | ||
153 | } | ||
154 | } ] | ||
155 | } | ||
156 | ] | ||
157 | } | ||
158 | ] | ||
159 | } | ||
160 | return VideoChannelSyncModel.unscoped().findAll(query) | ||
161 | } | ||
162 | |||
163 | toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync { | ||
164 | return { | ||
165 | id: this.id, | ||
166 | state: { | ||
167 | id: this.state, | ||
168 | label: VIDEO_CHANNEL_SYNC_STATE[this.state] | ||
169 | }, | ||
170 | externalChannelUrl: this.externalChannelUrl, | ||
171 | createdAt: this.createdAt.toISOString(), | ||
172 | channel: this.VideoChannel.toFormattedSummaryJSON(), | ||
173 | lastSyncAt: this.lastSyncAt?.toISOString() | ||
174 | } | ||
175 | } | ||
176 | } | ||
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index 1d8296060..b8e941623 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { WhereOptions } from 'sequelize' | 1 | import { Op, WhereOptions } from 'sequelize' |
2 | import { | 2 | import { |
3 | AfterUpdate, | 3 | AfterUpdate, |
4 | AllowNull, | 4 | AllowNull, |
@@ -161,6 +161,28 @@ export class VideoImportModel extends Model<Partial<AttributesOnly<VideoImportMo | |||
161 | ]).then(([ total, data ]) => ({ total, data })) | 161 | ]).then(([ total, data ]) => ({ total, data })) |
162 | } | 162 | } |
163 | 163 | ||
164 | static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> { | ||
165 | const element = await VideoImportModel.unscoped().findOne({ | ||
166 | where: { | ||
167 | targetUrl, | ||
168 | state: { | ||
169 | [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ] | ||
170 | } | ||
171 | }, | ||
172 | include: [ | ||
173 | { | ||
174 | model: VideoModel, | ||
175 | required: true, | ||
176 | where: { | ||
177 | channelId | ||
178 | } | ||
179 | } | ||
180 | ] | ||
181 | }) | ||
182 | |||
183 | return !!element | ||
184 | } | ||
185 | |||
164 | getTargetIdentifier () { | 186 | getTargetIdentifier () { |
165 | return this.targetUrl || this.magnetUri || this.torrentName | 187 | return this.targetUrl || this.magnetUri || this.torrentName |
166 | } | 188 | } |
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 2f9f553ab..d67e51123 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts | |||
@@ -1,7 +1,8 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import { omit } from 'lodash' | 4 | import { merge, omit } from 'lodash' |
5 | import { CustomConfig, HttpStatusCode } from '@shared/models' | ||
5 | import { | 6 | import { |
6 | cleanupTests, | 7 | cleanupTests, |
7 | createSingleServer, | 8 | createSingleServer, |
@@ -11,7 +12,6 @@ import { | |||
11 | PeerTubeServer, | 12 | PeerTubeServer, |
12 | setAccessTokensToServers | 13 | setAccessTokensToServers |
13 | } from '@shared/server-commands' | 14 | } from '@shared/server-commands' |
14 | import { CustomConfig, HttpStatusCode } from '@shared/models' | ||
15 | 15 | ||
16 | describe('Test config API validators', function () { | 16 | describe('Test config API validators', function () { |
17 | const path = '/api/v1/config/custom' | 17 | const path = '/api/v1/config/custom' |
@@ -162,6 +162,10 @@ describe('Test config API validators', function () { | |||
162 | torrent: { | 162 | torrent: { |
163 | enabled: false | 163 | enabled: false |
164 | } | 164 | } |
165 | }, | ||
166 | videoChannelSynchronization: { | ||
167 | enabled: false, | ||
168 | maxPerUser: 10 | ||
165 | } | 169 | } |
166 | }, | 170 | }, |
167 | trending: { | 171 | trending: { |
@@ -346,7 +350,26 @@ describe('Test config API validators', function () { | |||
346 | }) | 350 | }) |
347 | }) | 351 | }) |
348 | 352 | ||
349 | it('Should success with the correct parameters', async function () { | 353 | it('Should fail with a disabled http upload & enabled sync', async function () { |
354 | const newUpdateParams: CustomConfig = merge({}, updateParams, { | ||
355 | import: { | ||
356 | videos: { | ||
357 | http: { enabled: false } | ||
358 | }, | ||
359 | videoChannelSynchronization: { enabled: true } | ||
360 | } | ||
361 | }) | ||
362 | |||
363 | await makePutBodyRequest({ | ||
364 | url: server.url, | ||
365 | path, | ||
366 | fields: newUpdateParams, | ||
367 | token: server.accessToken, | ||
368 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
369 | }) | ||
370 | }) | ||
371 | |||
372 | it('Should succeed with the correct parameters', async function () { | ||
350 | await makePutBodyRequest({ | 373 | await makePutBodyRequest({ |
351 | url: server.url, | 374 | url: server.url, |
352 | path, | 375 | path, |
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index a27bc8509..5f1168b53 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts | |||
@@ -27,6 +27,7 @@ import './video-channels' | |||
27 | import './video-comments' | 27 | import './video-comments' |
28 | import './video-files' | 28 | import './video-files' |
29 | import './video-imports' | 29 | import './video-imports' |
30 | import './video-channel-syncs' | ||
30 | import './video-playlists' | 31 | import './video-playlists' |
31 | import './video-source' | 32 | import './video-source' |
32 | import './video-studio' | 33 | import './video-studio' |
diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts index deb4a7aa3..f64eafc18 100644 --- a/server/tests/api/check-params/upload-quota.ts +++ b/server/tests/api/check-params/upload-quota.ts | |||
@@ -70,7 +70,7 @@ describe('Test upload quota', function () { | |||
70 | }) | 70 | }) |
71 | 71 | ||
72 | it('Should fail to import with HTTP/Torrent/magnet', async function () { | 72 | it('Should fail to import with HTTP/Torrent/magnet', async function () { |
73 | this.timeout(120000) | 73 | this.timeout(120_000) |
74 | 74 | ||
75 | const baseAttributes = { | 75 | const baseAttributes = { |
76 | channelId: server.store.channel.id, | 76 | channelId: server.store.channel.id, |
diff --git a/server/tests/api/check-params/video-channel-syncs.ts b/server/tests/api/check-params/video-channel-syncs.ts new file mode 100644 index 000000000..bcd8984df --- /dev/null +++ b/server/tests/api/check-params/video-channel-syncs.ts | |||
@@ -0,0 +1,318 @@ | |||
1 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared' | ||
2 | import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models' | ||
3 | import { | ||
4 | ChannelSyncsCommand, | ||
5 | createSingleServer, | ||
6 | makePostBodyRequest, | ||
7 | PeerTubeServer, | ||
8 | setAccessTokensToServers, | ||
9 | setDefaultVideoChannel | ||
10 | } from '@shared/server-commands' | ||
11 | |||
12 | describe('Test video channel sync API validator', () => { | ||
13 | const path = '/api/v1/video-channel-syncs' | ||
14 | let server: PeerTubeServer | ||
15 | let command: ChannelSyncsCommand | ||
16 | let rootChannelId: number | ||
17 | let rootChannelSyncId: number | ||
18 | const userInfo = { | ||
19 | accessToken: '', | ||
20 | username: 'user1', | ||
21 | id: -1, | ||
22 | channelId: -1, | ||
23 | syncId: -1 | ||
24 | } | ||
25 | |||
26 | async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> { | ||
27 | try { | ||
28 | await server.config.disableChannelSync() | ||
29 | await callback() | ||
30 | } finally { | ||
31 | await server.config.enableChannelSync() | ||
32 | } | ||
33 | } | ||
34 | |||
35 | async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> { | ||
36 | const origConfig = await server.config.getCustomConfig() | ||
37 | |||
38 | await server.config.updateExistingSubConfig({ | ||
39 | newConfig: { | ||
40 | import: { | ||
41 | videoChannelSynchronization: { | ||
42 | maxPerUser: maxSync | ||
43 | } | ||
44 | } | ||
45 | } | ||
46 | }) | ||
47 | |||
48 | try { | ||
49 | await callback() | ||
50 | } finally { | ||
51 | await server.config.updateCustomConfig({ newCustomConfig: origConfig }) | ||
52 | } | ||
53 | } | ||
54 | |||
55 | before(async function () { | ||
56 | this.timeout(30_000) | ||
57 | |||
58 | server = await createSingleServer(1) | ||
59 | |||
60 | await setAccessTokensToServers([ server ]) | ||
61 | await setDefaultVideoChannel([ server ]) | ||
62 | |||
63 | command = server.channelSyncs | ||
64 | |||
65 | rootChannelId = server.store.channel.id | ||
66 | |||
67 | { | ||
68 | userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) | ||
69 | |||
70 | const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken }) | ||
71 | userInfo.id = userId | ||
72 | userInfo.channelId = videoChannels[0].id | ||
73 | } | ||
74 | |||
75 | await server.config.enableChannelSync() | ||
76 | }) | ||
77 | |||
78 | describe('When creating a sync', function () { | ||
79 | let baseCorrectParams: VideoChannelSyncCreate | ||
80 | |||
81 | before(function () { | ||
82 | baseCorrectParams = { | ||
83 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
84 | videoChannelId: rootChannelId | ||
85 | } | ||
86 | }) | ||
87 | |||
88 | it('Should fail when sync is disabled', async function () { | ||
89 | await withChannelSyncDisabled(async () => { | ||
90 | await command.create({ | ||
91 | token: server.accessToken, | ||
92 | attributes: baseCorrectParams, | ||
93 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
94 | }) | ||
95 | }) | ||
96 | }) | ||
97 | |||
98 | it('Should fail with nothing', async function () { | ||
99 | const fields = {} | ||
100 | await makePostBodyRequest({ | ||
101 | url: server.url, | ||
102 | path, | ||
103 | token: server.accessToken, | ||
104 | fields, | ||
105 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
106 | }) | ||
107 | }) | ||
108 | |||
109 | it('Should fail with no authentication', async function () { | ||
110 | await command.create({ | ||
111 | token: null, | ||
112 | attributes: baseCorrectParams, | ||
113 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
114 | }) | ||
115 | }) | ||
116 | |||
117 | it('Should fail without a target url', async function () { | ||
118 | const attributes: VideoChannelSyncCreate = { | ||
119 | ...baseCorrectParams, | ||
120 | externalChannelUrl: null | ||
121 | } | ||
122 | await command.create({ | ||
123 | token: server.accessToken, | ||
124 | attributes, | ||
125 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
126 | }) | ||
127 | }) | ||
128 | |||
129 | it('Should fail without a channelId', async function () { | ||
130 | const attributes: VideoChannelSyncCreate = { | ||
131 | ...baseCorrectParams, | ||
132 | videoChannelId: null | ||
133 | } | ||
134 | await command.create({ | ||
135 | token: server.accessToken, | ||
136 | attributes, | ||
137 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
138 | }) | ||
139 | }) | ||
140 | |||
141 | it('Should fail with a channelId refering nothing', async function () { | ||
142 | const attributes: VideoChannelSyncCreate = { | ||
143 | ...baseCorrectParams, | ||
144 | videoChannelId: 42 | ||
145 | } | ||
146 | await command.create({ | ||
147 | token: server.accessToken, | ||
148 | attributes, | ||
149 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
150 | }) | ||
151 | }) | ||
152 | |||
153 | it('Should fail to create a sync when the user does not own the channel', async function () { | ||
154 | await command.create({ | ||
155 | token: userInfo.accessToken, | ||
156 | attributes: baseCorrectParams, | ||
157 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
158 | }) | ||
159 | }) | ||
160 | |||
161 | it('Should succeed to create a sync with root and for another user\'s channel', async function () { | ||
162 | const { videoChannelSync } = await command.create({ | ||
163 | token: server.accessToken, | ||
164 | attributes: { | ||
165 | ...baseCorrectParams, | ||
166 | videoChannelId: userInfo.channelId | ||
167 | }, | ||
168 | expectedStatus: HttpStatusCode.OK_200 | ||
169 | }) | ||
170 | userInfo.syncId = videoChannelSync.id | ||
171 | }) | ||
172 | |||
173 | it('Should succeed with the correct parameters', async function () { | ||
174 | const { videoChannelSync } = await command.create({ | ||
175 | token: server.accessToken, | ||
176 | attributes: baseCorrectParams, | ||
177 | expectedStatus: HttpStatusCode.OK_200 | ||
178 | }) | ||
179 | rootChannelSyncId = videoChannelSync.id | ||
180 | }) | ||
181 | |||
182 | it('Should fail when the user exceeds allowed number of synchronizations', async function () { | ||
183 | await withMaxSyncsPerUser(1, async () => { | ||
184 | await command.create({ | ||
185 | token: server.accessToken, | ||
186 | attributes: { | ||
187 | ...baseCorrectParams, | ||
188 | videoChannelId: userInfo.channelId | ||
189 | }, | ||
190 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
191 | }) | ||
192 | }) | ||
193 | }) | ||
194 | }) | ||
195 | |||
196 | describe('When listing my channel syncs', function () { | ||
197 | const myPath = '/api/v1/accounts/root/video-channel-syncs' | ||
198 | |||
199 | it('Should fail with a bad start pagination', async function () { | ||
200 | await checkBadStartPagination(server.url, myPath, server.accessToken) | ||
201 | }) | ||
202 | |||
203 | it('Should fail with a bad count pagination', async function () { | ||
204 | await checkBadCountPagination(server.url, myPath, server.accessToken) | ||
205 | }) | ||
206 | |||
207 | it('Should fail with an incorrect sort', async function () { | ||
208 | await checkBadSortPagination(server.url, myPath, server.accessToken) | ||
209 | }) | ||
210 | |||
211 | it('Should succeed with the correct parameters', async function () { | ||
212 | await command.listByAccount({ | ||
213 | accountName: 'root', | ||
214 | token: server.accessToken, | ||
215 | expectedStatus: HttpStatusCode.OK_200 | ||
216 | }) | ||
217 | }) | ||
218 | |||
219 | it('Should fail with no authentication', async function () { | ||
220 | await command.listByAccount({ | ||
221 | accountName: 'root', | ||
222 | token: null, | ||
223 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
224 | }) | ||
225 | }) | ||
226 | |||
227 | it('Should fail when a simple user lists another user\'s synchronizations', async function () { | ||
228 | await command.listByAccount({ | ||
229 | accountName: 'root', | ||
230 | token: userInfo.accessToken, | ||
231 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
232 | }) | ||
233 | }) | ||
234 | |||
235 | it('Should succeed when root lists another user\'s synchronizations', async function () { | ||
236 | await command.listByAccount({ | ||
237 | accountName: userInfo.username, | ||
238 | token: server.accessToken, | ||
239 | expectedStatus: HttpStatusCode.OK_200 | ||
240 | }) | ||
241 | }) | ||
242 | |||
243 | it('Should succeed even with synchronization disabled', async function () { | ||
244 | await withChannelSyncDisabled(async function () { | ||
245 | await command.listByAccount({ | ||
246 | accountName: 'root', | ||
247 | token: server.accessToken, | ||
248 | expectedStatus: HttpStatusCode.OK_200 | ||
249 | }) | ||
250 | }) | ||
251 | }) | ||
252 | }) | ||
253 | |||
254 | describe('When triggering deletion', function () { | ||
255 | it('should fail with no authentication', async function () { | ||
256 | await command.delete({ | ||
257 | channelSyncId: userInfo.syncId, | ||
258 | token: null, | ||
259 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
260 | }) | ||
261 | }) | ||
262 | |||
263 | it('Should fail when channelSyncId does not refer to any sync', async function () { | ||
264 | await command.delete({ | ||
265 | channelSyncId: 42, | ||
266 | token: server.accessToken, | ||
267 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
268 | }) | ||
269 | }) | ||
270 | |||
271 | it('Should fail when sync is not owned by the user', async function () { | ||
272 | await command.delete({ | ||
273 | channelSyncId: rootChannelSyncId, | ||
274 | token: userInfo.accessToken, | ||
275 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
276 | }) | ||
277 | }) | ||
278 | |||
279 | it('Should succeed when root delete a sync they do not own', async function () { | ||
280 | await command.delete({ | ||
281 | channelSyncId: userInfo.syncId, | ||
282 | token: server.accessToken, | ||
283 | expectedStatus: HttpStatusCode.NO_CONTENT_204 | ||
284 | }) | ||
285 | }) | ||
286 | |||
287 | it('should succeed when user delete a sync they own', async function () { | ||
288 | const { videoChannelSync } = await command.create({ | ||
289 | attributes: { | ||
290 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
291 | videoChannelId: userInfo.channelId | ||
292 | }, | ||
293 | token: server.accessToken, | ||
294 | expectedStatus: HttpStatusCode.OK_200 | ||
295 | }) | ||
296 | |||
297 | await command.delete({ | ||
298 | channelSyncId: videoChannelSync.id, | ||
299 | token: server.accessToken, | ||
300 | expectedStatus: HttpStatusCode.NO_CONTENT_204 | ||
301 | }) | ||
302 | }) | ||
303 | |||
304 | it('Should succeed even when synchronization is disabled', async function () { | ||
305 | await withChannelSyncDisabled(async function () { | ||
306 | await command.delete({ | ||
307 | channelSyncId: rootChannelSyncId, | ||
308 | token: server.accessToken, | ||
309 | expectedStatus: HttpStatusCode.NO_CONTENT_204 | ||
310 | }) | ||
311 | }) | ||
312 | }) | ||
313 | }) | ||
314 | |||
315 | after(async function () { | ||
316 | await server?.kill() | ||
317 | }) | ||
318 | }) | ||
diff --git a/server/tests/api/check-params/video-channels.ts b/server/tests/api/check-params/video-channels.ts index 5c2650fac..337ea1dd4 100644 --- a/server/tests/api/check-params/video-channels.ts +++ b/server/tests/api/check-params/video-channels.ts | |||
@@ -3,8 +3,8 @@ | |||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { omit } from 'lodash' | 5 | import { omit } from 'lodash' |
6 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' | 6 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared' |
7 | import { buildAbsoluteFixturePath } from '@shared/core-utils' | 7 | import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils' |
8 | import { HttpStatusCode, VideoChannelUpdate } from '@shared/models' | 8 | import { HttpStatusCode, VideoChannelUpdate } from '@shared/models' |
9 | import { | 9 | import { |
10 | ChannelsCommand, | 10 | ChannelsCommand, |
@@ -23,7 +23,13 @@ const expect = chai.expect | |||
23 | describe('Test video channels API validator', function () { | 23 | describe('Test video channels API validator', function () { |
24 | const videoChannelPath = '/api/v1/video-channels' | 24 | const videoChannelPath = '/api/v1/video-channels' |
25 | let server: PeerTubeServer | 25 | let server: PeerTubeServer |
26 | let accessTokenUser: string | 26 | const userInfo = { |
27 | accessToken: '', | ||
28 | channelName: 'fake_channel', | ||
29 | id: -1, | ||
30 | videoQuota: -1, | ||
31 | videoQuotaDaily: -1 | ||
32 | } | ||
27 | let command: ChannelsCommand | 33 | let command: ChannelsCommand |
28 | 34 | ||
29 | // --------------------------------------------------------------- | 35 | // --------------------------------------------------------------- |
@@ -35,14 +41,15 @@ describe('Test video channels API validator', function () { | |||
35 | 41 | ||
36 | await setAccessTokensToServers([ server ]) | 42 | await setAccessTokensToServers([ server ]) |
37 | 43 | ||
38 | const user = { | 44 | const userCreds = { |
39 | username: 'fake', | 45 | username: 'fake', |
40 | password: 'fake_password' | 46 | password: 'fake_password' |
41 | } | 47 | } |
42 | 48 | ||
43 | { | 49 | { |
44 | await server.users.create({ username: user.username, password: user.password }) | 50 | const user = await server.users.create({ username: userCreds.username, password: userCreds.password }) |
45 | accessTokenUser = await server.login.getAccessToken(user) | 51 | userInfo.id = user.id |
52 | userInfo.accessToken = await server.login.getAccessToken(userCreds) | ||
46 | } | 53 | } |
47 | 54 | ||
48 | command = server.channels | 55 | command = server.channels |
@@ -191,7 +198,7 @@ describe('Test video channels API validator', function () { | |||
191 | await makePutBodyRequest({ | 198 | await makePutBodyRequest({ |
192 | url: server.url, | 199 | url: server.url, |
193 | path, | 200 | path, |
194 | token: accessTokenUser, | 201 | token: userInfo.accessToken, |
195 | fields: baseCorrectParams, | 202 | fields: baseCorrectParams, |
196 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | 203 | expectedStatus: HttpStatusCode.FORBIDDEN_403 |
197 | }) | 204 | }) |
@@ -339,7 +346,7 @@ describe('Test video channels API validator', function () { | |||
339 | }) | 346 | }) |
340 | 347 | ||
341 | it('Should fail with a another user', async function () { | 348 | it('Should fail with a another user', async function () { |
342 | await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | 349 | await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
343 | }) | 350 | }) |
344 | 351 | ||
345 | it('Should succeed with the correct params', async function () { | 352 | it('Should succeed with the correct params', async function () { |
@@ -347,13 +354,122 @@ describe('Test video channels API validator', function () { | |||
347 | }) | 354 | }) |
348 | }) | 355 | }) |
349 | 356 | ||
357 | describe('When triggering full synchronization', function () { | ||
358 | |||
359 | it('Should fail when HTTP upload is disabled', async function () { | ||
360 | await server.config.disableImports() | ||
361 | |||
362 | await command.importVideos({ | ||
363 | channelName: 'super_channel', | ||
364 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
365 | token: server.accessToken, | ||
366 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
367 | }) | ||
368 | |||
369 | await server.config.enableImports() | ||
370 | }) | ||
371 | |||
372 | it('Should fail when externalChannelUrl is not provided', async function () { | ||
373 | await command.importVideos({ | ||
374 | channelName: 'super_channel', | ||
375 | externalChannelUrl: null, | ||
376 | token: server.accessToken, | ||
377 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
378 | }) | ||
379 | }) | ||
380 | |||
381 | it('Should fail when externalChannelUrl is malformed', async function () { | ||
382 | await command.importVideos({ | ||
383 | channelName: 'super_channel', | ||
384 | externalChannelUrl: 'not-a-url', | ||
385 | token: server.accessToken, | ||
386 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
387 | }) | ||
388 | }) | ||
389 | |||
390 | it('Should fail with no authentication', async function () { | ||
391 | await command.importVideos({ | ||
392 | channelName: 'super_channel', | ||
393 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
394 | token: null, | ||
395 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
396 | }) | ||
397 | }) | ||
398 | |||
399 | it('Should fail when sync is not owned by the user', async function () { | ||
400 | await command.importVideos({ | ||
401 | channelName: 'super_channel', | ||
402 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
403 | token: userInfo.accessToken, | ||
404 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
405 | }) | ||
406 | }) | ||
407 | |||
408 | it('Should fail when the user has no quota', async function () { | ||
409 | await server.users.update({ | ||
410 | userId: userInfo.id, | ||
411 | videoQuota: 0 | ||
412 | }) | ||
413 | |||
414 | await command.importVideos({ | ||
415 | channelName: 'fake_channel', | ||
416 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
417 | token: userInfo.accessToken, | ||
418 | expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 | ||
419 | }) | ||
420 | |||
421 | await server.users.update({ | ||
422 | userId: userInfo.id, | ||
423 | videoQuota: userInfo.videoQuota | ||
424 | }) | ||
425 | }) | ||
426 | |||
427 | it('Should fail when the user has no daily quota', async function () { | ||
428 | await server.users.update({ | ||
429 | userId: userInfo.id, | ||
430 | videoQuotaDaily: 0 | ||
431 | }) | ||
432 | |||
433 | await command.importVideos({ | ||
434 | channelName: 'fake_channel', | ||
435 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
436 | token: userInfo.accessToken, | ||
437 | expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 | ||
438 | }) | ||
439 | |||
440 | await server.users.update({ | ||
441 | userId: userInfo.id, | ||
442 | videoQuotaDaily: userInfo.videoQuotaDaily | ||
443 | }) | ||
444 | }) | ||
445 | |||
446 | it('Should succeed when sync is run by its owner', async function () { | ||
447 | if (!areHttpImportTestsDisabled()) return | ||
448 | |||
449 | await command.importVideos({ | ||
450 | channelName: 'fake_channel', | ||
451 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
452 | token: userInfo.accessToken | ||
453 | }) | ||
454 | }) | ||
455 | |||
456 | it('Should succeed when sync is run with root and for another user\'s channel', async function () { | ||
457 | if (!areHttpImportTestsDisabled()) return | ||
458 | |||
459 | await command.importVideos({ | ||
460 | channelName: 'fake_channel', | ||
461 | externalChannelUrl: FIXTURE_URLS.youtubeChannel | ||
462 | }) | ||
463 | }) | ||
464 | }) | ||
465 | |||
350 | describe('When deleting a video channel', function () { | 466 | describe('When deleting a video channel', function () { |
351 | it('Should fail with a non authenticated user', async function () { | 467 | it('Should fail with a non authenticated user', async function () { |
352 | await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) | 468 | await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) |
353 | }) | 469 | }) |
354 | 470 | ||
355 | it('Should fail with another authenticated user', async function () { | 471 | it('Should fail with another authenticated user', async function () { |
356 | await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | 472 | await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
357 | }) | 473 | }) |
358 | 474 | ||
359 | it('Should fail with an unknown video channel id', async function () { | 475 | it('Should fail with an unknown video channel id', async function () { |
diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts index 4439810e8..5cdd0d925 100644 --- a/server/tests/api/check-params/video-imports.ts +++ b/server/tests/api/check-params/video-imports.ts | |||
@@ -88,7 +88,13 @@ describe('Test video imports API validator', function () { | |||
88 | 88 | ||
89 | it('Should fail with nothing', async function () { | 89 | it('Should fail with nothing', async function () { |
90 | const fields = {} | 90 | const fields = {} |
91 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | 91 | await makePostBodyRequest({ |
92 | url: server.url, | ||
93 | path, | ||
94 | token: server.accessToken, | ||
95 | fields, | ||
96 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
97 | }) | ||
92 | }) | 98 | }) |
93 | 99 | ||
94 | it('Should fail without a target url', async function () { | 100 | it('Should fail without a target url', async function () { |
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index efc57b345..fc8711161 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts | |||
@@ -368,6 +368,10 @@ const newCustomConfig: CustomConfig = { | |||
368 | torrent: { | 368 | torrent: { |
369 | enabled: false | 369 | enabled: false |
370 | } | 370 | } |
371 | }, | ||
372 | videoChannelSynchronization: { | ||
373 | enabled: false, | ||
374 | maxPerUser: 10 | ||
371 | } | 375 | } |
372 | }, | 376 | }, |
373 | trending: { | 377 | trending: { |
diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts new file mode 100644 index 000000000..f7540e1ba --- /dev/null +++ b/server/tests/api/videos/channel-import-videos.ts | |||
@@ -0,0 +1,50 @@ | |||
1 | import { expect } from 'chai' | ||
2 | import { FIXTURE_URLS } from '@server/tests/shared' | ||
3 | import { areHttpImportTestsDisabled } from '@shared/core-utils' | ||
4 | import { | ||
5 | createSingleServer, | ||
6 | getServerImportConfig, | ||
7 | PeerTubeServer, | ||
8 | setAccessTokensToServers, | ||
9 | setDefaultVideoChannel, | ||
10 | waitJobs | ||
11 | } from '@shared/server-commands' | ||
12 | |||
13 | describe('Test videos import in a channel', function () { | ||
14 | if (areHttpImportTestsDisabled()) return | ||
15 | |||
16 | function runSuite (mode: 'youtube-dl' | 'yt-dlp') { | ||
17 | |||
18 | describe('Import using ' + mode, function () { | ||
19 | let server: PeerTubeServer | ||
20 | |||
21 | before(async function () { | ||
22 | this.timeout(120_000) | ||
23 | |||
24 | server = await createSingleServer(1, getServerImportConfig(mode)) | ||
25 | |||
26 | await setAccessTokensToServers([ server ]) | ||
27 | await setDefaultVideoChannel([ server ]) | ||
28 | |||
29 | await server.config.enableChannelSync() | ||
30 | }) | ||
31 | |||
32 | it('Should import a whole channel', async function () { | ||
33 | this.timeout(240_000) | ||
34 | |||
35 | await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel }) | ||
36 | await waitJobs(server) | ||
37 | |||
38 | const videos = await server.videos.listByChannel({ handle: server.store.channel.name }) | ||
39 | expect(videos.total).to.equal(2) | ||
40 | }) | ||
41 | |||
42 | after(async function () { | ||
43 | await server?.kill() | ||
44 | }) | ||
45 | }) | ||
46 | } | ||
47 | |||
48 | runSuite('yt-dlp') | ||
49 | runSuite('youtube-dl') | ||
50 | }) | ||
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index a0b6b01cf..266155297 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts | |||
@@ -4,6 +4,8 @@ import './single-server' | |||
4 | import './video-captions' | 4 | import './video-captions' |
5 | import './video-change-ownership' | 5 | import './video-change-ownership' |
6 | import './video-channels' | 6 | import './video-channels' |
7 | import './channel-import-videos' | ||
8 | import './video-channel-syncs' | ||
7 | import './video-comments' | 9 | import './video-comments' |
8 | import './video-description' | 10 | import './video-description' |
9 | import './video-files' | 11 | import './video-files' |
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts new file mode 100644 index 000000000..229c01f68 --- /dev/null +++ b/server/tests/api/videos/video-channel-syncs.ts | |||
@@ -0,0 +1,226 @@ | |||
1 | import 'mocha' | ||
2 | import { expect } from 'chai' | ||
3 | import { FIXTURE_URLS } from '@server/tests/shared' | ||
4 | import { areHttpImportTestsDisabled } from '@shared/core-utils' | ||
5 | import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models' | ||
6 | import { | ||
7 | ChannelSyncsCommand, | ||
8 | createSingleServer, | ||
9 | getServerImportConfig, | ||
10 | PeerTubeServer, | ||
11 | setAccessTokensToServers, | ||
12 | setDefaultAccountAvatar, | ||
13 | setDefaultChannelAvatar, | ||
14 | setDefaultVideoChannel, | ||
15 | waitJobs | ||
16 | } from '@shared/server-commands' | ||
17 | |||
18 | describe('Test channel synchronizations', function () { | ||
19 | if (areHttpImportTestsDisabled()) return | ||
20 | |||
21 | function runSuite (mode: 'youtube-dl' | 'yt-dlp') { | ||
22 | |||
23 | describe('Sync using ' + mode, function () { | ||
24 | let server: PeerTubeServer | ||
25 | let command: ChannelSyncsCommand | ||
26 | let startTestDate: Date | ||
27 | const userInfo = { | ||
28 | accessToken: '', | ||
29 | username: 'user1', | ||
30 | channelName: 'user1_channel', | ||
31 | channelId: -1, | ||
32 | syncId: -1 | ||
33 | } | ||
34 | |||
35 | async function changeDateForSync (channelSyncId: number, newDate: string) { | ||
36 | await server.sql.updateQuery( | ||
37 | `UPDATE "videoChannelSync" ` + | ||
38 | `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` + | ||
39 | `WHERE id=${channelSyncId}` | ||
40 | ) | ||
41 | } | ||
42 | |||
43 | before(async function () { | ||
44 | this.timeout(120_000) | ||
45 | |||
46 | startTestDate = new Date() | ||
47 | |||
48 | server = await createSingleServer(1, getServerImportConfig(mode)) | ||
49 | |||
50 | await setAccessTokensToServers([ server ]) | ||
51 | await setDefaultVideoChannel([ server ]) | ||
52 | await setDefaultChannelAvatar([ server ]) | ||
53 | await setDefaultAccountAvatar([ server ]) | ||
54 | |||
55 | await server.config.enableChannelSync() | ||
56 | |||
57 | command = server.channelSyncs | ||
58 | |||
59 | { | ||
60 | userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) | ||
61 | |||
62 | const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken }) | ||
63 | userInfo.channelId = videoChannels[0].id | ||
64 | } | ||
65 | }) | ||
66 | |||
67 | it('Should fetch the latest channel videos of a remote channel', async function () { | ||
68 | this.timeout(120_000) | ||
69 | |||
70 | { | ||
71 | const { video } = await server.imports.importVideo({ | ||
72 | attributes: { | ||
73 | channelId: server.store.channel.id, | ||
74 | privacy: VideoPrivacy.PUBLIC, | ||
75 | targetUrl: FIXTURE_URLS.youtube | ||
76 | } | ||
77 | }) | ||
78 | |||
79 | expect(video.name).to.equal('small video - youtube') | ||
80 | |||
81 | const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) | ||
82 | expect(total).to.equal(1) | ||
83 | } | ||
84 | |||
85 | const { videoChannelSync } = await command.create({ | ||
86 | attributes: { | ||
87 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
88 | videoChannelId: server.store.channel.id | ||
89 | }, | ||
90 | token: server.accessToken, | ||
91 | expectedStatus: HttpStatusCode.OK_200 | ||
92 | }) | ||
93 | |||
94 | // Ensure any missing video not already fetched will be considered as new | ||
95 | await changeDateForSync(videoChannelSync.id, '1970-01-01') | ||
96 | |||
97 | await server.debug.sendCommand({ | ||
98 | body: { | ||
99 | command: 'process-video-channel-sync-latest' | ||
100 | } | ||
101 | }) | ||
102 | |||
103 | { | ||
104 | await waitJobs(server) | ||
105 | |||
106 | const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) | ||
107 | expect(total).to.equal(2) | ||
108 | expect(data[0].name).to.equal('test') | ||
109 | } | ||
110 | }) | ||
111 | |||
112 | it('Should add another synchronization', async function () { | ||
113 | const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar' | ||
114 | |||
115 | const { videoChannelSync } = await command.create({ | ||
116 | attributes: { | ||
117 | externalChannelUrl, | ||
118 | videoChannelId: server.store.channel.id | ||
119 | }, | ||
120 | token: server.accessToken, | ||
121 | expectedStatus: HttpStatusCode.OK_200 | ||
122 | }) | ||
123 | |||
124 | expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl) | ||
125 | expect(videoChannelSync.channel).to.include({ | ||
126 | id: server.store.channel.id, | ||
127 | name: 'root_channel' | ||
128 | }) | ||
129 | expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN) | ||
130 | expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date()) | ||
131 | }) | ||
132 | |||
133 | it('Should add a synchronization for another user', async function () { | ||
134 | const { videoChannelSync } = await command.create({ | ||
135 | attributes: { | ||
136 | externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', | ||
137 | videoChannelId: userInfo.channelId | ||
138 | }, | ||
139 | token: userInfo.accessToken | ||
140 | }) | ||
141 | userInfo.syncId = videoChannelSync.id | ||
142 | }) | ||
143 | |||
144 | it('Should not import a channel if not asked', async function () { | ||
145 | await waitJobs(server) | ||
146 | |||
147 | const { data } = await command.listByAccount({ accountName: userInfo.username }) | ||
148 | |||
149 | expect(data[0].state).to.contain({ | ||
150 | id: VideoChannelSyncState.WAITING_FIRST_RUN, | ||
151 | label: 'Waiting first run' | ||
152 | }) | ||
153 | }) | ||
154 | |||
155 | it('Should only fetch the videos newer than the creation date', async function () { | ||
156 | this.timeout(120_000) | ||
157 | |||
158 | await changeDateForSync(userInfo.syncId, '2019-03-01') | ||
159 | |||
160 | await server.debug.sendCommand({ | ||
161 | body: { | ||
162 | command: 'process-video-channel-sync-latest' | ||
163 | } | ||
164 | }) | ||
165 | |||
166 | await waitJobs(server) | ||
167 | |||
168 | const { data, total } = await server.videos.listByChannel({ | ||
169 | handle: userInfo.channelName, | ||
170 | include: VideoInclude.NOT_PUBLISHED_STATE | ||
171 | }) | ||
172 | |||
173 | expect(total).to.equal(1) | ||
174 | expect(data[0].name).to.equal('test') | ||
175 | }) | ||
176 | |||
177 | it('Should list channel synchronizations', async function () { | ||
178 | // Root | ||
179 | { | ||
180 | const { total, data } = await command.listByAccount({ accountName: 'root' }) | ||
181 | expect(total).to.equal(2) | ||
182 | |||
183 | expect(data[0]).to.deep.contain({ | ||
184 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
185 | state: { | ||
186 | id: VideoChannelSyncState.SYNCED, | ||
187 | label: 'Synchronized' | ||
188 | } | ||
189 | }) | ||
190 | |||
191 | expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate) | ||
192 | |||
193 | expect(data[0].channel).to.contain({ id: server.store.channel.id }) | ||
194 | expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' }) | ||
195 | } | ||
196 | |||
197 | // User | ||
198 | { | ||
199 | const { total, data } = await command.listByAccount({ accountName: userInfo.username }) | ||
200 | expect(total).to.equal(1) | ||
201 | expect(data[0]).to.deep.contain({ | ||
202 | externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', | ||
203 | state: { | ||
204 | id: VideoChannelSyncState.SYNCED, | ||
205 | label: 'Synchronized' | ||
206 | } | ||
207 | }) | ||
208 | } | ||
209 | }) | ||
210 | |||
211 | it('Should remove user\'s channel synchronizations', async function () { | ||
212 | await command.delete({ channelSyncId: userInfo.syncId }) | ||
213 | |||
214 | const { total } = await command.listByAccount({ accountName: userInfo.username }) | ||
215 | expect(total).to.equal(0) | ||
216 | }) | ||
217 | |||
218 | after(async function () { | ||
219 | await server?.kill() | ||
220 | }) | ||
221 | }) | ||
222 | } | ||
223 | |||
224 | runSuite('youtube-dl') | ||
225 | runSuite('yt-dlp') | ||
226 | }) | ||
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index 603e2d234..a487062a2 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts | |||
@@ -12,6 +12,7 @@ import { | |||
12 | createMultipleServers, | 12 | createMultipleServers, |
13 | createSingleServer, | 13 | createSingleServer, |
14 | doubleFollow, | 14 | doubleFollow, |
15 | getServerImportConfig, | ||
15 | PeerTubeServer, | 16 | PeerTubeServer, |
16 | setAccessTokensToServers, | 17 | setAccessTokensToServers, |
17 | setDefaultVideoChannel, | 18 | setDefaultVideoChannel, |
@@ -84,24 +85,9 @@ describe('Test video imports', function () { | |||
84 | let servers: PeerTubeServer[] = [] | 85 | let servers: PeerTubeServer[] = [] |
85 | 86 | ||
86 | before(async function () { | 87 | before(async function () { |
87 | this.timeout(30_000) | 88 | this.timeout(60_000) |
88 | 89 | ||
89 | // Run servers | 90 | servers = await createMultipleServers(2, getServerImportConfig(mode)) |
90 | servers = await createMultipleServers(2, { | ||
91 | import: { | ||
92 | videos: { | ||
93 | http: { | ||
94 | youtube_dl_release: { | ||
95 | url: mode === 'youtube-dl' | ||
96 | ? 'https://yt-dl.org/downloads/latest/youtube-dl' | ||
97 | : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases', | ||
98 | |||
99 | name: mode | ||
100 | } | ||
101 | } | ||
102 | } | ||
103 | } | ||
104 | }) | ||
105 | 91 | ||
106 | await setAccessTokensToServers(servers) | 92 | await setAccessTokensToServers(servers) |
107 | await setDefaultVideoChannel(servers) | 93 | await setDefaultVideoChannel(servers) |
diff --git a/server/tests/shared/tests.ts b/server/tests/shared/tests.ts index 3abaf833d..e67a294dc 100644 --- a/server/tests/shared/tests.ts +++ b/server/tests/shared/tests.ts | |||
@@ -16,6 +16,8 @@ const FIXTURE_URLS = { | |||
16 | */ | 16 | */ |
17 | youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4', | 17 | youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4', |
18 | 18 | ||
19 | youtubeChannel: 'https://youtube.com/channel/UCtnlZdXv3-xQzxiqfn6cjIA', | ||
20 | |||
19 | // eslint-disable-next-line max-len | 21 | // eslint-disable-next-line max-len |
20 | magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4', | 22 | magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4', |
21 | 23 | ||
diff --git a/server/types/express.d.ts b/server/types/express.d.ts index 8f8c65102..27d60da72 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts | |||
@@ -8,6 +8,7 @@ import { | |||
8 | MActorFollowActorsDefault, | 8 | MActorFollowActorsDefault, |
9 | MActorUrl, | 9 | MActorUrl, |
10 | MChannelBannerAccountDefault, | 10 | MChannelBannerAccountDefault, |
11 | MChannelSyncChannel, | ||
11 | MStreamingPlaylist, | 12 | MStreamingPlaylist, |
12 | MVideoChangeOwnershipFull, | 13 | MVideoChangeOwnershipFull, |
13 | MVideoFile, | 14 | MVideoFile, |
@@ -145,6 +146,7 @@ declare module 'express' { | |||
145 | videoStreamingPlaylist?: MStreamingPlaylist | 146 | videoStreamingPlaylist?: MStreamingPlaylist |
146 | 147 | ||
147 | videoChannel?: MChannelBannerAccountDefault | 148 | videoChannel?: MChannelBannerAccountDefault |
149 | videoChannelSync?: MChannelSyncChannel | ||
148 | 150 | ||
149 | videoPlaylistFull?: MVideoPlaylistFull | 151 | videoPlaylistFull?: MVideoPlaylistFull |
150 | videoPlaylistSummary?: MVideoPlaylistFullSummary | 152 | videoPlaylistSummary?: MVideoPlaylistFullSummary |
@@ -194,6 +196,7 @@ declare module 'express' { | |||
194 | plugin?: MPlugin | 196 | plugin?: MPlugin |
195 | 197 | ||
196 | localViewerFull?: MLocalVideoViewerWithWatchSections | 198 | localViewerFull?: MLocalVideoViewerWithWatchSections |
199 | |||
197 | } | 200 | } |
198 | } | 201 | } |
199 | } | 202 | } |
diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts index fdf8e1ddb..940f0ac0d 100644 --- a/server/types/models/video/index.ts +++ b/server/types/models/video/index.ts | |||
@@ -8,6 +8,7 @@ export * from './video' | |||
8 | export * from './video-blacklist' | 8 | export * from './video-blacklist' |
9 | export * from './video-caption' | 9 | export * from './video-caption' |
10 | export * from './video-change-ownership' | 10 | export * from './video-change-ownership' |
11 | export * from './video-channel-sync' | ||
11 | export * from './video-channels' | 12 | export * from './video-channels' |
12 | export * from './video-comment' | 13 | export * from './video-comment' |
13 | export * from './video-file' | 14 | export * from './video-file' |
diff --git a/server/types/models/video/video-channel-sync.ts b/server/types/models/video/video-channel-sync.ts new file mode 100644 index 000000000..429ab70b0 --- /dev/null +++ b/server/types/models/video/video-channel-sync.ts | |||
@@ -0,0 +1,17 @@ | |||
1 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
2 | import { FunctionProperties, PickWith } from '@shared/typescript-utils' | ||
3 | import { MChannelAccountDefault, MChannelFormattable } from './video-channels' | ||
4 | |||
5 | type Use<K extends keyof VideoChannelSyncModel, M> = PickWith<VideoChannelSyncModel, K, M> | ||
6 | |||
7 | export type MChannelSync = Omit<VideoChannelSyncModel, 'VideoChannel'> | ||
8 | |||
9 | export type MChannelSyncChannel = | ||
10 | MChannelSync & | ||
11 | Use<'VideoChannel', MChannelAccountDefault> & | ||
12 | FunctionProperties<VideoChannelSyncModel> | ||
13 | |||
14 | export type MChannelSyncFormattable = | ||
15 | FunctionProperties<MChannelSyncChannel> & | ||
16 | Use<'VideoChannel', MChannelFormattable> & | ||
17 | MChannelSync | ||