aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorFlorent <florent.git@zeteo.me>2022-08-10 09:53:39 +0200
committerGitHub <noreply@github.com>2022-08-10 09:53:39 +0200
commit2a491182e483b97afb1b65c908b23cb48d591807 (patch)
treeec13503216ad72a3ea8f1ce3b659899f8167fb47 /server
parent06ac128958c489efe1008eeca1df683819bd2f18 (diff)
downloadPeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.gz
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.zst
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.zip
Channel sync (#5135)
* Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/accounts.ts28
-rw-r--r--server/controllers/api/config.ts4
-rw-r--r--server/controllers/api/index.ts2
-rw-r--r--server/controllers/api/server/debug.ts4
-rw-r--r--server/controllers/api/video-channel-sync.ts76
-rw-r--r--server/controllers/api/video-channel.ts28
-rw-r--r--server/controllers/api/videos/import.ts318
-rw-r--r--server/helpers/audit-logger.ts17
-rw-r--r--server/helpers/custom-validators/video-channel-syncs.ts6
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts27
-rw-r--r--server/helpers/youtube-dl/youtube-dl-info-builder.ts4
-rw-r--r--server/helpers/youtube-dl/youtube-dl-wrapper.ts20
-rw-r--r--server/initializers/checker-after-init.ts7
-rw-r--r--server/initializers/checker-before-init.ts2
-rw-r--r--server/initializers/config.ts9
-rw-r--r--server/initializers/constants.ts29
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/initializers/migrations/0730-video-channel-sync.ts36
-rw-r--r--server/lib/job-queue/handlers/after-video-channel-import.ts37
-rw-r--r--server/lib/job-queue/handlers/video-channel-import.ts36
-rw-r--r--server/lib/job-queue/handlers/video-import.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts19
-rw-r--r--server/lib/schedulers/video-channel-sync-latest-scheduler.ts61
-rw-r--r--server/lib/server-config-manager.ts3
-rw-r--r--server/lib/sync-channel.ts81
-rw-r--r--server/lib/video-import.ts308
-rw-r--r--server/middlewares/validators/config.ts11
-rw-r--r--server/middlewares/validators/sort.ts2
-rw-r--r--server/middlewares/validators/videos/index.ts1
-rw-r--r--server/middlewares/validators/videos/video-channel-sync.ts66
-rw-r--r--server/middlewares/validators/videos/video-channels.ts55
-rw-r--r--server/models/utils.ts11
-rw-r--r--server/models/video/video-channel-sync.ts176
-rw-r--r--server/models/video/video-import.ts24
-rw-r--r--server/tests/api/check-params/config.ts29
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/upload-quota.ts2
-rw-r--r--server/tests/api/check-params/video-channel-syncs.ts318
-rw-r--r--server/tests/api/check-params/video-channels.ts134
-rw-r--r--server/tests/api/check-params/video-imports.ts8
-rw-r--r--server/tests/api/server/config.ts4
-rw-r--r--server/tests/api/videos/channel-import-videos.ts50
-rw-r--r--server/tests/api/videos/index.ts2
-rw-r--r--server/tests/api/videos/video-channel-syncs.ts226
-rw-r--r--server/tests/api/videos/video-imports.ts22
-rw-r--r--server/tests/shared/tests.ts2
-rw-r--r--server/types/express.d.ts3
-rw-r--r--server/types/models/video/index.ts1
-rw-r--r--server/types/models/video/video-channel-sync.ts17
49 files changed, 2012 insertions, 339 deletions
diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts
index 66cdaab82..7a530cde5 100644
--- a/server/controllers/api/accounts.ts
+++ b/server/controllers/api/accounts.ts
@@ -25,8 +25,10 @@ import {
25 accountsFollowersSortValidator, 25 accountsFollowersSortValidator,
26 accountsSortValidator, 26 accountsSortValidator,
27 ensureAuthUserOwnsAccountValidator, 27 ensureAuthUserOwnsAccountValidator,
28 ensureCanManageUser,
28 videoChannelsSortValidator, 29 videoChannelsSortValidator,
29 videoChannelStatsValidator, 30 videoChannelStatsValidator,
31 videoChannelSyncsSortValidator,
30 videosSortValidator 32 videosSortValidator
31} from '../../middlewares/validators' 33} from '../../middlewares/validators'
32import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists' 34import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
@@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate'
35import { VideoModel } from '../../models/video/video' 37import { VideoModel } from '../../models/video/video'
36import { VideoChannelModel } from '../../models/video/video-channel' 38import { VideoChannelModel } from '../../models/video/video-channel'
37import { VideoPlaylistModel } from '../../models/video/video-playlist' 39import { VideoPlaylistModel } from '../../models/video/video-playlist'
40import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
38 41
39const accountsRouter = express.Router() 42const accountsRouter = express.Router()
40 43
@@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels',
72 asyncMiddleware(listAccountChannels) 75 asyncMiddleware(listAccountChannels)
73) 76)
74 77
78accountsRouter.get('/:accountName/video-channel-syncs',
79 authenticate,
80 asyncMiddleware(accountNameWithHostGetValidator),
81 ensureCanManageUser,
82 paginationValidator,
83 videoChannelSyncsSortValidator,
84 setDefaultSort,
85 setDefaultPagination,
86 asyncMiddleware(listAccountChannelsSync)
87)
88
75accountsRouter.get('/:accountName/video-playlists', 89accountsRouter.get('/:accountName/video-playlists',
76 optionalAuthenticate, 90 optionalAuthenticate,
77 asyncMiddleware(accountNameWithHostGetValidator), 91 asyncMiddleware(accountNameWithHostGetValidator),
@@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response)
146 return res.json(getFormattedObjects(resultList.data, resultList.total)) 160 return res.json(getFormattedObjects(resultList.data, resultList.total))
147} 161}
148 162
163async function listAccountChannelsSync (req: express.Request, res: express.Response) {
164 const options = {
165 accountId: res.locals.account.id,
166 start: req.query.start,
167 count: req.query.count,
168 sort: req.query.sort,
169 search: req.query.search
170 }
171
172 const resultList = await VideoChannelSyncModel.listByAccountForAPI(options)
173
174 return res.json(getFormattedObjects(resultList.data, resultList.total))
175}
176
149async function listAccountPlaylists (req: express.Request, res: express.Response) { 177async function listAccountPlaylists (req: express.Request, res: express.Response) {
150 const serverActor = await getServerActor() 178 const serverActor = await getServerActor()
151 179
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index ff2fa9d86..f0fb43071 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -273,6 +273,10 @@ function customConfig (): CustomConfig {
273 torrent: { 273 torrent: {
274 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED 274 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
275 } 275 }
276 },
277 videoChannelSynchronization: {
278 enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
279 maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
276 } 280 }
277 }, 281 },
278 trending: { 282 trending: {
diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts
index d1d4ef765..8c8ebd061 100644
--- a/server/controllers/api/index.ts
+++ b/server/controllers/api/index.ts
@@ -20,6 +20,7 @@ import { usersRouter } from './users'
20import { videoChannelRouter } from './video-channel' 20import { videoChannelRouter } from './video-channel'
21import { videoPlaylistRouter } from './video-playlist' 21import { videoPlaylistRouter } from './video-playlist'
22import { videosRouter } from './videos' 22import { videosRouter } from './videos'
23import { videoChannelSyncRouter } from './video-channel-sync'
23 24
24const apiRouter = express.Router() 25const apiRouter = express.Router()
25 26
@@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter)
43apiRouter.use('/users', usersRouter) 44apiRouter.use('/users', usersRouter)
44apiRouter.use('/accounts', accountsRouter) 45apiRouter.use('/accounts', accountsRouter)
45apiRouter.use('/video-channels', videoChannelRouter) 46apiRouter.use('/video-channels', videoChannelRouter)
47apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
46apiRouter.use('/video-playlists', videoPlaylistRouter) 48apiRouter.use('/video-playlists', videoPlaylistRouter)
47apiRouter.use('/videos', videosRouter) 49apiRouter.use('/videos', videosRouter)
48apiRouter.use('/jobs', jobsRouter) 50apiRouter.use('/jobs', jobsRouter)
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index e09510dc3..4e5333782 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models'
7import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' 7import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
8import { UserRight } from '../../../../shared/models/users' 8import { UserRight } from '../../../../shared/models/users'
9import { authenticate, ensureUserHasRight } from '../../../middlewares' 9import { authenticate, ensureUserHasRight } from '../../../middlewares'
10import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
10 11
11const debugRouter = express.Router() 12const debugRouter = express.Router()
12 13
@@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) {
43 const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = { 44 const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
44 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), 45 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
45 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), 46 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
46 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats() 47 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
48 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
47 } 49 }
48 50
49 await processors[body.command]() 51 await processors[body.command]()
diff --git a/server/controllers/api/video-channel-sync.ts b/server/controllers/api/video-channel-sync.ts
new file mode 100644
index 000000000..c2770b8e4
--- /dev/null
+++ b/server/controllers/api/video-channel-sync.ts
@@ -0,0 +1,76 @@
1import express from 'express'
2import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger'
3import { logger } from '@server/helpers/logger'
4import {
5 asyncMiddleware,
6 asyncRetryTransactionMiddleware,
7 authenticate,
8 ensureCanManageChannel as ensureCanManageSyncedChannel,
9 ensureSyncExists,
10 ensureSyncIsEnabled,
11 videoChannelSyncValidator
12} from '@server/middlewares'
13import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
14import { MChannelSyncFormattable } from '@server/types/models'
15import { HttpStatusCode, VideoChannelSyncState } from '@shared/models'
16
17const videoChannelSyncRouter = express.Router()
18const auditLogger = auditLoggerFactory('channel-syncs')
19
20videoChannelSyncRouter.post('/',
21 authenticate,
22 ensureSyncIsEnabled,
23 asyncMiddleware(videoChannelSyncValidator),
24 ensureCanManageSyncedChannel,
25 asyncRetryTransactionMiddleware(createVideoChannelSync)
26)
27
28videoChannelSyncRouter.delete('/:id',
29 authenticate,
30 asyncMiddleware(ensureSyncExists),
31 ensureCanManageSyncedChannel,
32 asyncRetryTransactionMiddleware(removeVideoChannelSync)
33)
34
35export { videoChannelSyncRouter }
36
37// ---------------------------------------------------------------------------
38
39async function createVideoChannelSync (req: express.Request, res: express.Response) {
40 const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({
41 externalChannelUrl: req.body.externalChannelUrl,
42 videoChannelId: req.body.videoChannelId,
43 state: VideoChannelSyncState.WAITING_FIRST_RUN
44 })
45
46 await syncCreated.save()
47 syncCreated.VideoChannel = res.locals.videoChannel
48
49 auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON()))
50
51 logger.info(
52 'Video synchronization for channel "%s" with external channel "%s" created.',
53 syncCreated.VideoChannel.name,
54 syncCreated.externalChannelUrl
55 )
56
57 return res.json({
58 videoChannelSync: syncCreated.toFormattedJSON()
59 })
60}
61
62async function removeVideoChannelSync (req: express.Request, res: express.Response) {
63 const syncInstance = res.locals.videoChannelSync
64
65 await syncInstance.destroy()
66
67 auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON()))
68
69 logger.info(
70 'Video synchronization for channel "%s" with external channel "%s" deleted.',
71 syncInstance.VideoChannel.name,
72 syncInstance.externalChannelUrl
73 )
74
75 return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
76}
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts
index 6b33e894d..89c7181bd 100644
--- a/server/controllers/api/video-channel.ts
+++ b/server/controllers/api/video-channel.ts
@@ -36,7 +36,9 @@ import {
36 videoPlaylistsSortValidator 36 videoPlaylistsSortValidator
37} from '../../middlewares' 37} from '../../middlewares'
38import { 38import {
39 ensureChannelOwnerCanUpload,
39 ensureIsLocalChannel, 40 ensureIsLocalChannel,
41 videoChannelImportVideosValidator,
40 videoChannelsFollowersSortValidator, 42 videoChannelsFollowersSortValidator,
41 videoChannelsListValidator, 43 videoChannelsListValidator,
42 videoChannelsNameWithHostValidator, 44 videoChannelsNameWithHostValidator,
@@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers',
161 asyncMiddleware(listVideoChannelFollowers) 163 asyncMiddleware(listVideoChannelFollowers)
162) 164)
163 165
166videoChannelRouter.post('/:nameWithHost/import-videos',
167 authenticate,
168 asyncMiddleware(videoChannelsNameWithHostValidator),
169 videoChannelImportVideosValidator,
170 ensureIsLocalChannel,
171 ensureCanManageChannel,
172 asyncMiddleware(ensureChannelOwnerCanUpload),
173 asyncMiddleware(importVideosInChannel)
174)
175
164// --------------------------------------------------------------------------- 176// ---------------------------------------------------------------------------
165 177
166export { 178export {
@@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res
404 416
405 return res.json(getFormattedObjects(resultList.data, resultList.total)) 417 return res.json(getFormattedObjects(resultList.data, resultList.total))
406} 418}
419
420async function importVideosInChannel (req: express.Request, res: express.Response) {
421 const { externalChannelUrl } = req.body
422
423 await JobQueue.Instance.createJob({
424 type: 'video-channel-import',
425 payload: {
426 externalChannelUrl,
427 videoChannelId: res.locals.videoChannel.id
428 }
429 })
430
431 logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl)
432
433 return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
434}
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 5a2e1006a..9d7b0260b 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,49 +1,20 @@
1import express from 'express' 1import express from 'express'
2import { move, readFile, remove } from 'fs-extra' 2import { move, readFile } from 'fs-extra'
3import { decode } from 'magnet-uri' 3import { decode } from 'magnet-uri'
4import parseTorrent, { Instance } from 'parse-torrent' 4import parseTorrent, { Instance } from 'parse-torrent'
5import { join } from 'path' 5import { join } from 'path'
6import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' 6import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
7import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' 7import { MThumbnail, MVideoThumbnail } from '@server/types/models'
8import { isResolvingToUnicastOnly } from '@server/helpers/dns' 8import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
9import { Hooks } from '@server/lib/plugins/hooks'
10import { ServerConfigManager } from '@server/lib/server-config-manager'
11import { setVideoTags } from '@server/lib/video'
12import { FilteredModelAttributes } from '@server/types'
13import {
14 MChannelAccountDefault,
15 MThumbnail,
16 MUser,
17 MVideoAccountDefault,
18 MVideoCaption,
19 MVideoTag,
20 MVideoThumbnail,
21 MVideoWithBlacklistLight
22} from '@server/types/models'
23import { MVideoImportFormattable } from '@server/types/models/video/video-import'
24import {
25 HttpStatusCode,
26 ServerErrorCode,
27 ThumbnailType,
28 VideoImportCreate,
29 VideoImportState,
30 VideoPrivacy,
31 VideoState
32} from '@shared/models'
33import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' 9import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
34import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
35import { isArray } from '../../../helpers/custom-validators/misc' 10import { isArray } from '../../../helpers/custom-validators/misc'
36import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' 11import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
37import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
38import { getSecureTorrentName } from '../../../helpers/utils' 13import { getSecureTorrentName } from '../../../helpers/utils'
39import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
40import { CONFIG } from '../../../initializers/config' 14import { CONFIG } from '../../../initializers/config'
41import { MIMETYPES } from '../../../initializers/constants' 15import { MIMETYPES } from '../../../initializers/constants'
42import { sequelizeTypescript } from '../../../initializers/database'
43import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
44import { JobQueue } from '../../../lib/job-queue/job-queue' 16import { JobQueue } from '../../../lib/job-queue/job-queue'
45import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail' 17import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
46import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
47import { 18import {
48 asyncMiddleware, 19 asyncMiddleware,
49 asyncRetryTransactionMiddleware, 20 asyncRetryTransactionMiddleware,
@@ -52,9 +23,6 @@ import {
52 videoImportCancelValidator, 23 videoImportCancelValidator,
53 videoImportDeleteValidator 24 videoImportDeleteValidator
54} from '../../../middlewares' 25} from '../../../middlewares'
55import { VideoModel } from '../../../models/video/video'
56import { VideoCaptionModel } from '../../../models/video/video-caption'
57import { VideoImportModel } from '../../../models/video/video-import'
58 26
59const auditLogger = auditLoggerFactory('video-imports') 27const auditLogger = auditLoggerFactory('video-imports')
60const videoImportsRouter = express.Router() 28const videoImportsRouter = express.Router()
@@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
68 authenticate, 36 authenticate,
69 reqVideoFileImport, 37 reqVideoFileImport,
70 asyncMiddleware(videoImportAddValidator), 38 asyncMiddleware(videoImportAddValidator),
71 asyncRetryTransactionMiddleware(addVideoImport) 39 asyncRetryTransactionMiddleware(handleVideoImport)
72) 40)
73 41
74videoImportsRouter.post('/imports/:id/cancel', 42videoImportsRouter.post('/imports/:id/cancel',
@@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
108 return res.sendStatus(HttpStatusCode.NO_CONTENT_204) 76 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
109} 77}
110 78
111function addVideoImport (req: express.Request, res: express.Response) { 79function handleVideoImport (req: express.Request, res: express.Response) {
112 if (req.body.targetUrl) return addYoutubeDLImport(req, res) 80 if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
113 81
114 const file = req.files?.['torrentfile']?.[0] 82 const file = req.files?.['torrentfile']?.[0]
115 if (req.body.magnetUri || file) return addTorrentImport(req, res, file) 83 if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
116} 84}
117 85
118async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { 86async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
119 const body: VideoImportCreate = req.body 87 const body: VideoImportCreate = req.body
120 const user = res.locals.oauth.token.User 88 const user = res.locals.oauth.token.User
121 89
@@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
135 videoName = result.name 103 videoName = result.name
136 } 104 }
137 105
138 const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName }) 106 const video = await buildVideoFromImport({
107 channelId: res.locals.videoChannel.id,
108 importData: { name: videoName },
109 importDataOverride: body,
110 importType: 'torrent'
111 })
139 112
140 const thumbnailModel = await processThumbnail(req, video) 113 const thumbnailModel = await processThumbnail(req, video)
141 const previewModel = await processPreview(req, video) 114 const previewModel = await processPreview(req, video)
142 115
143 const videoImport = await insertIntoDB({ 116 const videoImport = await insertFromImportIntoDB({
144 video, 117 video,
145 thumbnailModel, 118 thumbnailModel,
146 previewModel, 119 previewModel,
@@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
155 } 128 }
156 }) 129 })
157 130
158 // Create job to import the video 131 const payload: VideoImportPayload = {
159 const payload = {
160 type: torrentfile 132 type: torrentfile
161 ? 'torrent-file' as 'torrent-file' 133 ? 'torrent-file'
162 : 'magnet-uri' as 'magnet-uri', 134 : 'magnet-uri',
163 videoImportId: videoImport.id, 135 videoImportId: videoImport.id,
164 magnetUri 136 preventException: false
165 } 137 }
166 await JobQueue.Instance.createJob({ type: 'video-import', payload }) 138 await JobQueue.Instance.createJob({ type: 'video-import', payload })
167 139
@@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
170 return res.json(videoImport.toFormattedJSON()).end() 142 return res.json(videoImport.toFormattedJSON()).end()
171} 143}
172 144
173async function addYoutubeDLImport (req: express.Request, res: express.Response) { 145function statusFromYtDlImportError (err: YoutubeDlImportError): number {
146 switch (err.code) {
147 case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
148 return HttpStatusCode.FORBIDDEN_403
149
150 case YoutubeDlImportError.CODE.FETCH_ERROR:
151 return HttpStatusCode.BAD_REQUEST_400
152
153 default:
154 return HttpStatusCode.INTERNAL_SERVER_ERROR_500
155 }
156}
157
158async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
174 const body: VideoImportCreate = req.body 159 const body: VideoImportCreate = req.body
175 const targetUrl = body.targetUrl 160 const targetUrl = body.targetUrl
176 const user = res.locals.oauth.token.User 161 const user = res.locals.oauth.token.User
177 162
178 const youtubeDL = new YoutubeDLWrapper(
179 targetUrl,
180 ServerConfigManager.Instance.getEnabledResolutions('vod'),
181 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
182 )
183
184 // Get video infos
185 let youtubeDLInfo: YoutubeDLInfo
186 try { 163 try {
187 youtubeDLInfo = await youtubeDL.getInfoForDownload() 164 const { job, videoImport } = await buildYoutubeDLImport({
165 targetUrl,
166 channel: res.locals.videoChannel,
167 importDataOverride: body,
168 thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
169 previewFilePath: req.files?.['previewfile']?.[0].path,
170 user
171 })
172 await JobQueue.Instance.createJob(job)
173
174 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
175
176 return res.json(videoImport.toFormattedJSON()).end()
188 } catch (err) { 177 } catch (err) {
189 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) 178 logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
190 179
191 return res.fail({ 180 return res.fail({
192 message: 'Cannot fetch remote information of this URL.', 181 message: err.message,
182 status: statusFromYtDlImportError(err),
193 data: { 183 data: {
194 targetUrl 184 targetUrl
195 } 185 }
196 }) 186 })
197 } 187 }
198
199 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
200 return res.fail({
201 status: HttpStatusCode.FORBIDDEN_403,
202 message: 'Cannot use non unicast IP as targetUrl.'
203 })
204 }
205
206 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
207
208 // Process video thumbnail from request.files
209 let thumbnailModel = await processThumbnail(req, video)
210
211 // Process video thumbnail from url if processing from request.files failed
212 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
213 try {
214 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
215 } catch (err) {
216 logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
217 }
218 }
219
220 // Process video preview from request.files
221 let previewModel = await processPreview(req, video)
222
223 // Process video preview from url if processing from request.files failed
224 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
225 try {
226 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
227 } catch (err) {
228 logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
229 }
230 }
231
232 const videoImport = await insertIntoDB({
233 video,
234 thumbnailModel,
235 previewModel,
236 videoChannel: res.locals.videoChannel,
237 tags: body.tags || youtubeDLInfo.tags,
238 user,
239 videoImportAttributes: {
240 targetUrl,
241 state: VideoImportState.PENDING,
242 userId: user.id
243 }
244 })
245
246 // Get video subtitles
247 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
248
249 let fileExt = `.${youtubeDLInfo.ext}`
250 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
251
252 // Create job to import the video
253 const payload = {
254 type: 'youtube-dl' as 'youtube-dl',
255 videoImportId: videoImport.id,
256 fileExt
257 }
258 await JobQueue.Instance.createJob({ type: 'video-import', payload })
259
260 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
261
262 return res.json(videoImport.toFormattedJSON()).end()
263}
264
265async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
266 let videoData = {
267 name: body.name || importData.name || 'Unknown name',
268 remote: false,
269 category: body.category || importData.category,
270 licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
271 language: body.language || importData.language,
272 commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
273 downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
274 waitTranscoding: body.waitTranscoding || false,
275 state: VideoState.TO_IMPORT,
276 nsfw: body.nsfw || importData.nsfw || false,
277 description: body.description || importData.description,
278 support: body.support || null,
279 privacy: body.privacy || VideoPrivacy.PRIVATE,
280 duration: 0, // duration will be set by the import job
281 channelId,
282 originallyPublishedAt: body.originallyPublishedAt
283 ? new Date(body.originallyPublishedAt)
284 : importData.originallyPublishedAt
285 }
286
287 videoData = await Hooks.wrapObject(
288 videoData,
289 body.targetUrl
290 ? 'filter:api.video.import-url.video-attribute.result'
291 : 'filter:api.video.import-torrent.video-attribute.result'
292 )
293
294 const video = new VideoModel(videoData)
295 video.url = getLocalVideoActivityPubUrl(video)
296
297 return video
298} 188}
299 189
300async function processThumbnail (req: express.Request, video: MVideoThumbnail) { 190async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
@@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
329 return undefined 219 return undefined
330} 220}
331 221
332async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
333 try {
334 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
335 } catch (err) {
336 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
337 return undefined
338 }
339}
340
341async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
342 try {
343 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
344 } catch (err) {
345 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
346 return undefined
347 }
348}
349
350async function insertIntoDB (parameters: {
351 video: MVideoThumbnail
352 thumbnailModel: MThumbnail
353 previewModel: MThumbnail
354 videoChannel: MChannelAccountDefault
355 tags: string[]
356 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
357 user: MUser
358}): Promise<MVideoImportFormattable> {
359 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
360
361 const videoImport = await sequelizeTypescript.transaction(async t => {
362 const sequelizeOptions = { transaction: t }
363
364 // Save video object in database
365 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
366 videoCreated.VideoChannel = videoChannel
367
368 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
369 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
370
371 await autoBlacklistVideoIfNeeded({
372 video: videoCreated,
373 user,
374 notify: false,
375 isRemote: false,
376 isNew: true,
377 transaction: t
378 })
379
380 await setVideoTags({ video: videoCreated, tags, transaction: t })
381
382 // Create video import object in database
383 const videoImport = await VideoImportModel.create(
384 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
385 sequelizeOptions
386 ) as MVideoImportFormattable
387 videoImport.Video = videoCreated
388
389 return videoImport
390 })
391
392 return videoImport
393}
394
395async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { 222async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
396 const torrentName = torrentfile.originalname 223 const torrentName = torrentfile.originalname
397 224
@@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
432function extractNameFromArray (name: string | string[]) { 259function extractNameFromArray (name: string | string[]) {
433 return isArray(name) ? name[0] : name 260 return isArray(name) ? name[0] : name
434} 261}
435
436async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
437 try {
438 const subtitles = await youtubeDL.getSubtitles()
439
440 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
441
442 for (const subtitle of subtitles) {
443 if (!await isVTTFileValid(subtitle.path)) {
444 await remove(subtitle.path)
445 continue
446 }
447
448 const videoCaption = new VideoCaptionModel({
449 videoId,
450 language: subtitle.language,
451 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
452 }) as MVideoCaption
453
454 // Move physical file
455 await moveAndProcessCaptionFile(subtitle, videoCaption)
456
457 await sequelizeTypescript.transaction(async t => {
458 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
459 })
460 }
461 } catch (err) {
462 logger.warn('Cannot get video subtitles.', { err })
463 }
464}
465
466async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
467 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
468 const uniqHosts = new Set(hosts)
469
470 for (const h of uniqHosts) {
471 if (await isResolvingToUnicastOnly(h) !== true) {
472 return false
473 }
474 }
475
476 return true
477}
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 076b7f11d..7e8a03e8f 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -5,7 +5,7 @@ import { chain } from 'lodash'
5import { join } from 'path' 5import { join } from 'path'
6import { addColors, config, createLogger, format, transports } from 'winston' 6import { addColors, config, createLogger, format, transports } from 'winston'
7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' 7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
8import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models' 8import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
9import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
10import { jsonLoggerFormat, labelFormatter } from './logger' 10import { jsonLoggerFormat, labelFormatter } from './logger'
11 11
@@ -260,6 +260,18 @@ class CustomConfigAuditView extends EntityAuditView {
260 } 260 }
261} 261}
262 262
263const channelSyncKeysToKeep = [
264 'id',
265 'externalChannelUrl',
266 'channel-id',
267 'channel-name'
268]
269class VideoChannelSyncAuditView extends EntityAuditView {
270 constructor (channelSync: VideoChannelSync) {
271 super(channelSyncKeysToKeep, 'channelSync', channelSync)
272 }
273}
274
263export { 275export {
264 getAuditIdFromRes, 276 getAuditIdFromRes,
265 277
@@ -270,5 +282,6 @@ export {
270 UserAuditView, 282 UserAuditView,
271 VideoAuditView, 283 VideoAuditView,
272 AbuseAuditView, 284 AbuseAuditView,
273 CustomConfigAuditView 285 CustomConfigAuditView,
286 VideoChannelSyncAuditView
274} 287}
diff --git a/server/helpers/custom-validators/video-channel-syncs.ts b/server/helpers/custom-validators/video-channel-syncs.ts
new file mode 100644
index 000000000..c5a9afa96
--- /dev/null
+++ b/server/helpers/custom-validators/video-channel-syncs.ts
@@ -0,0 +1,6 @@
1import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
2import { exists } from './misc'
3
4export function isVideoChannelSyncStateValid (value: any) {
5 return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
6}
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 13c990a1e..5a87b99b4 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -87,6 +87,7 @@ export class YoutubeDLCLI {
87 return result.concat([ 87 return result.concat([
88 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', 88 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
89 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats 89 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
90 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
90 'best' // Ultimate fallback 91 'best' // Ultimate fallback
91 ]).join('/') 92 ]).join('/')
92 } 93 }
@@ -103,11 +104,14 @@ export class YoutubeDLCLI {
103 timeout?: number 104 timeout?: number
104 additionalYoutubeDLArgs?: string[] 105 additionalYoutubeDLArgs?: string[]
105 }) { 106 }) {
107 let args = options.additionalYoutubeDLArgs || []
108 args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
109
106 return this.run({ 110 return this.run({
107 url: options.url, 111 url: options.url,
108 processOptions: options.processOptions, 112 processOptions: options.processOptions,
109 timeout: options.timeout, 113 timeout: options.timeout,
110 args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ]) 114 args
111 }) 115 })
112 } 116 }
113 117
@@ -129,6 +133,25 @@ export class YoutubeDLCLI {
129 : info 133 : info
130 } 134 }
131 135
136 getListInfo (options: {
137 url: string
138 latestVideosCount?: number
139 processOptions: execa.NodeOptions
140 }): Promise<{ upload_date: string, webpage_url: string }[]> {
141 const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
142
143 if (options.latestVideosCount !== undefined) {
144 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
145 }
146
147 return this.getInfo({
148 url: options.url,
149 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
150 processOptions: options.processOptions,
151 additionalYoutubeDLArgs
152 })
153 }
154
132 async getSubs (options: { 155 async getSubs (options: {
133 url: string 156 url: string
134 format: 'vtt' 157 format: 'vtt'
@@ -175,7 +198,7 @@ export class YoutubeDLCLI {
175 198
176 const output = await subProcess 199 const output = await subProcess
177 200
178 logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() }) 201 logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
179 202
180 return output.stdout 203 return output.stdout
181 ? output.stdout.trim().split(/\r?\n/) 204 ? output.stdout.trim().split(/\r?\n/)
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
index 71572f292..303e4051f 100644
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
@@ -13,6 +13,7 @@ type YoutubeDLInfo = {
13 thumbnailUrl?: string 13 thumbnailUrl?: string
14 ext?: string 14 ext?: string
15 originallyPublishedAt?: Date 15 originallyPublishedAt?: Date
16 webpageUrl?: string
16 17
17 urls?: string[] 18 urls?: string[]
18} 19}
@@ -81,7 +82,8 @@ class YoutubeDLInfoBuilder {
81 thumbnailUrl: obj.thumbnail || undefined, 82 thumbnailUrl: obj.thumbnail || undefined,
82 urls: this.buildAvailableUrl(obj), 83 urls: this.buildAvailableUrl(obj),
83 originallyPublishedAt: this.buildOriginallyPublishedAt(obj), 84 originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
84 ext: obj.ext 85 ext: obj.ext,
86 webpageUrl: obj.webpage_url
85 } 87 }
86 } 88 }
87 89
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index 176cf3b69..7cd5e3310 100644
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -46,6 +46,24 @@ class YoutubeDLWrapper {
46 return infoBuilder.getInfo() 46 return infoBuilder.getInfo()
47 } 47 }
48 48
49 async getInfoForListImport (options: {
50 latestVideosCount?: number
51 }) {
52 const youtubeDL = await YoutubeDLCLI.safeGet()
53
54 const list = await youtubeDL.getListInfo({
55 url: this.url,
56 latestVideosCount: options.latestVideosCount,
57 processOptions
58 })
59
60 return list.map(info => {
61 const infoBuilder = new YoutubeDLInfoBuilder(info)
62
63 return infoBuilder.getInfo()
64 })
65 }
66
49 async getSubtitles (): Promise<YoutubeDLSubs> { 67 async getSubtitles (): Promise<YoutubeDLSubs> {
50 const cwd = CONFIG.STORAGE.TMP_DIR 68 const cwd = CONFIG.STORAGE.TMP_DIR
51 69
@@ -103,7 +121,7 @@ class YoutubeDLWrapper {
103 121
104 return remove(path) 122 return remove(path)
105 }) 123 })
106 .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) 124 .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
107 125
108 throw err 126 throw err
109 } 127 }
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index f0f16d9bd..74c82541e 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -48,6 +48,7 @@ function checkConfig () {
48 checkRemoteRedundancyConfig() 48 checkRemoteRedundancyConfig()
49 checkStorageConfig() 49 checkStorageConfig()
50 checkTranscodingConfig() 50 checkTranscodingConfig()
51 checkImportConfig()
51 checkBroadcastMessageConfig() 52 checkBroadcastMessageConfig()
52 checkSearchConfig() 53 checkSearchConfig()
53 checkLiveConfig() 54 checkLiveConfig()
@@ -200,6 +201,12 @@ function checkTranscodingConfig () {
200 } 201 }
201} 202}
202 203
204function checkImportConfig () {
205 if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) {
206 throw new Error('You need to enable HTTP import to allow synchronization')
207 }
208}
209
203function checkBroadcastMessageConfig () { 210function checkBroadcastMessageConfig () {
204 if (CONFIG.BROADCAST_MESSAGE.ENABLED) { 211 if (CONFIG.BROADCAST_MESSAGE.ENABLED) {
205 const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL 212 const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index f4057b81b..3188903be 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -32,6 +32,8 @@ function checkMissedConfig () {
32 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', 32 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
33 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled', 33 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
34 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', 34 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
35 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
36 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
35 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', 37 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
36 'client.videos.miniature.display_author_avatar', 38 'client.videos.miniature.display_author_avatar',
37 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', 39 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 1a0b8942c..2c92bea22 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -398,6 +398,14 @@ const CONFIG = {
398 TORRENT: { 398 TORRENT: {
399 get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') } 399 get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
400 } 400 }
401 },
402 VIDEO_CHANNEL_SYNCHRONIZATION: {
403 get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') },
404 get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') },
405 get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
406 get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
407 return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
408 }
401 } 409 }
402 }, 410 },
403 AUTO_BLACKLIST: { 411 AUTO_BLACKLIST: {
@@ -499,6 +507,7 @@ const CONFIG = {
499 get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') } 507 get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
500 } 508 }
501 } 509 }
510
502} 511}
503 512
504function registerConfigChangedHandler (fun: Function) { 513function registerConfigChangedHandler (fun: Function) {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5a5f2d666..697a64d42 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
6import { 6import {
7 AbuseState, 7 AbuseState,
8 JobType, 8 JobType,
9 VideoChannelSyncState,
9 VideoImportState, 10 VideoImportState,
10 VideoPrivacy, 11 VideoPrivacy,
11 VideoRateType, 12 VideoRateType,
@@ -24,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
24 25
25// --------------------------------------------------------------------------- 26// ---------------------------------------------------------------------------
26 27
27const LAST_MIGRATION_VERSION = 725 28const LAST_MIGRATION_VERSION = 730
28 29
29// --------------------------------------------------------------------------- 30// ---------------------------------------------------------------------------
30 31
@@ -64,6 +65,7 @@ const SORTABLE_COLUMNS = {
64 JOBS: [ 'createdAt' ], 65 JOBS: [ 'createdAt' ],
65 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], 66 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
66 VIDEO_IMPORTS: [ 'createdAt' ], 67 VIDEO_IMPORTS: [ 'createdAt' ],
68 VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ],
67 69
68 VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ], 70 VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
69 VIDEO_COMMENTS: [ 'createdAt' ], 71 VIDEO_COMMENTS: [ 'createdAt' ],
@@ -156,6 +158,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
156 'video-live-ending': 1, 158 'video-live-ending': 1,
157 'video-studio-edition': 1, 159 'video-studio-edition': 1,
158 'manage-video-torrent': 1, 160 'manage-video-torrent': 1,
161 'video-channel-import': 1,
162 'after-video-channel-import': 1,
159 'move-to-object-storage': 3, 163 'move-to-object-storage': 3,
160 'notify': 1, 164 'notify': 1,
161 'federate-video': 1 165 'federate-video': 1
@@ -178,6 +182,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
178 'video-studio-edition': 1, 182 'video-studio-edition': 1,
179 'manage-video-torrent': 1, 183 'manage-video-torrent': 1,
180 'move-to-object-storage': 1, 184 'move-to-object-storage': 1,
185 'video-channel-import': 1,
186 'after-video-channel-import': 1,
181 'notify': 5, 187 'notify': 5,
182 'federate-video': 3 188 'federate-video': 3
183} 189}
@@ -199,9 +205,11 @@ const JOB_TTL: { [id in JobType]: number } = {
199 'video-redundancy': 1000 * 3600 * 3, // 3 hours 205 'video-redundancy': 1000 * 3600 * 3, // 3 hours
200 'video-live-ending': 1000 * 60 * 10, // 10 minutes 206 'video-live-ending': 1000 * 60 * 10, // 10 minutes
201 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours 207 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
208 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
209 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
210 'after-video-channel-import': 60000 * 5, // 5 minutes
202 'notify': 60000 * 5, // 5 minutes 211 'notify': 60000 * 5, // 5 minutes
203 'federate-video': 60000 * 5, // 5 minutes 212 'federate-video': 60000 * 5 // 5 minutes
204 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
205} 213}
206const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = { 214const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
207 'videos-views-stats': { 215 'videos-views-stats': {
@@ -246,7 +254,8 @@ const SCHEDULER_INTERVALS_MS = {
246 REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day 254 REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
247 REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day 255 REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
248 UPDATE_INBOX_STATS: 1000 * 60, // 1 minute 256 UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
249 REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour 257 REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
258 CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
250} 259}
251 260
252// --------------------------------------------------------------------------- 261// ---------------------------------------------------------------------------
@@ -276,8 +285,12 @@ const CONSTRAINTS_FIELDS = {
276 NAME: { min: 1, max: 120 }, // Length 285 NAME: { min: 1, max: 120 }, // Length
277 DESCRIPTION: { min: 3, max: 1000 }, // Length 286 DESCRIPTION: { min: 3, max: 1000 }, // Length
278 SUPPORT: { min: 3, max: 1000 }, // Length 287 SUPPORT: { min: 3, max: 1000 }, // Length
288 EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length
279 URL: { min: 3, max: 2000 } // Length 289 URL: { min: 3, max: 2000 } // Length
280 }, 290 },
291 VIDEO_CHANNEL_SYNCS: {
292 EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length
293 },
281 VIDEO_CAPTIONS: { 294 VIDEO_CAPTIONS: {
282 CAPTION_FILE: { 295 CAPTION_FILE: {
283 EXTNAME: [ '.vtt', '.srt' ], 296 EXTNAME: [ '.vtt', '.srt' ],
@@ -478,6 +491,13 @@ const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
478 [VideoImportState.PROCESSING]: 'Processing' 491 [VideoImportState.PROCESSING]: 'Processing'
479} 492}
480 493
494const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = {
495 [VideoChannelSyncState.FAILED]: 'Failed',
496 [VideoChannelSyncState.SYNCED]: 'Synchronized',
497 [VideoChannelSyncState.PROCESSING]: 'Processing',
498 [VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run'
499}
500
481const ABUSE_STATES: { [ id in AbuseState ]: string } = { 501const ABUSE_STATES: { [ id in AbuseState ]: string } = {
482 [AbuseState.PENDING]: 'Pending', 502 [AbuseState.PENDING]: 'Pending',
483 [AbuseState.REJECTED]: 'Rejected', 503 [AbuseState.REJECTED]: 'Rejected',
@@ -1005,6 +1025,7 @@ export {
1005 JOB_COMPLETED_LIFETIME, 1025 JOB_COMPLETED_LIFETIME,
1006 HTTP_SIGNATURE, 1026 HTTP_SIGNATURE,
1007 VIDEO_IMPORT_STATES, 1027 VIDEO_IMPORT_STATES,
1028 VIDEO_CHANNEL_SYNC_STATE,
1008 VIEW_LIFETIME, 1029 VIEW_LIFETIME,
1009 CONTACT_FORM_LIFETIME, 1030 CONTACT_FORM_LIFETIME,
1010 VIDEO_PLAYLIST_PRIVACIES, 1031 VIDEO_PLAYLIST_PRIVACIES,
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 91286241b..f55f40df0 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -50,6 +50,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
50import { VideoTagModel } from '../models/video/video-tag' 50import { VideoTagModel } from '../models/video/video-tag'
51import { VideoViewModel } from '../models/view/video-view' 51import { VideoViewModel } from '../models/view/video-view'
52import { CONFIG } from './config' 52import { CONFIG } from './config'
53import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
53 54
54require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 55require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
55 56
@@ -153,7 +154,8 @@ async function initDatabaseModels (silent: boolean) {
153 VideoTrackerModel, 154 VideoTrackerModel,
154 PluginModel, 155 PluginModel,
155 ActorCustomPageModel, 156 ActorCustomPageModel,
156 VideoJobInfoModel 157 VideoJobInfoModel,
158 VideoChannelSyncModel
157 ]) 159 ])
158 160
159 // Check extensions exist in the database 161 // Check extensions exist in the database
diff --git a/server/initializers/migrations/0730-video-channel-sync.ts b/server/initializers/migrations/0730-video-channel-sync.ts
new file mode 100644
index 000000000..a2fe8211f
--- /dev/null
+++ b/server/initializers/migrations/0730-video-channel-sync.ts
@@ -0,0 +1,36 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction
5 queryInterface: Sequelize.QueryInterface
6 sequelize: Sequelize.Sequelize
7 db: any
8}): Promise<void> {
9 const query = `
10 CREATE TABLE IF NOT EXISTS "videoChannelSync" (
11 "id" SERIAL,
12 "externalChannelUrl" VARCHAR(2000) NOT NULL DEFAULT NULL,
13 "videoChannelId" INTEGER NOT NULL REFERENCES "videoChannel" ("id")
14 ON DELETE CASCADE
15 ON UPDATE CASCADE,
16 "state" INTEGER NOT NULL DEFAULT 1,
17 "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
18 "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
19 "lastSyncAt" TIMESTAMP WITH TIME ZONE,
20 PRIMARY KEY ("id")
21 );
22 `
23 await utils.sequelize.query(query, { transaction: utils.transaction })
24}
25
26async function down (utils: {
27 queryInterface: Sequelize.QueryInterface
28 transaction: Sequelize.Transaction
29}) {
30 await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction })
31}
32
33export {
34 up,
35 down
36}
diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts
new file mode 100644
index 000000000..ffdd8c5b5
--- /dev/null
+++ b/server/lib/job-queue/handlers/after-video-channel-import.ts
@@ -0,0 +1,37 @@
1import { Job } from 'bullmq'
2import { logger } from '@server/helpers/logger'
3import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
4import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
5
6export async function processAfterVideoChannelImport (job: Job) {
7 const payload = job.data as AfterVideoChannelImportPayload
8 if (!payload.channelSyncId) return
9
10 logger.info('Processing after video channel import in job %s.', job.id)
11
12 const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
13 if (!sync) {
14 logger.error('Unknown sync id %d.', payload.channelSyncId)
15 return
16 }
17
18 const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
19
20 let errors = 0
21 let successes = 0
22
23 for (const value of Object.values(childrenValues)) {
24 if (value.resultType === 'success') successes++
25 else if (value.resultType === 'error') errors++
26 }
27
28 if (errors > 0) {
29 sync.state = VideoChannelSyncState.FAILED
30 logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
31 } else {
32 sync.state = VideoChannelSyncState.SYNCED
33 logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
34 }
35
36 await sync.save()
37}
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
new file mode 100644
index 000000000..9bdb2d269
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -0,0 +1,36 @@
1import { Job } from 'bullmq'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { synchronizeChannel } from '@server/lib/sync-channel'
5import { VideoChannelModel } from '@server/models/video/video-channel'
6import { VideoChannelImportPayload } from '@shared/models'
7
8export async function processVideoChannelImport (job: Job) {
9 const payload = job.data as VideoChannelImportPayload
10
11 logger.info('Processing video channel import in job %s.', job.id)
12
13 // Channel import requires only http upload to be allowed
14 if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
15 logger.error('Cannot import channel as the HTTP upload is disabled')
16 return
17 }
18
19 if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
20 logger.error('Cannot import channel as the synchronization is disabled')
21 return
22 }
23
24 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
25
26 try {
27 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
28
29 await synchronizeChannel({
30 channel: videoChannel,
31 externalChannelUrl: payload.externalChannelUrl
32 })
33 } catch (err) {
34 logger.error(`Failed to import channel ${videoChannel.name}`, { err })
35 }
36}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index f4629159c..9901b878c 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
8import { Hooks } from '@server/lib/plugins/hooks' 8import { Hooks } from '@server/lib/plugins/hooks'
9import { ServerConfigManager } from '@server/lib/server-config-manager' 9import { ServerConfigManager } from '@server/lib/server-config-manager'
10import { isAbleToUploadVideo } from '@server/lib/user' 10import { isAbleToUploadVideo } from '@server/lib/user'
11import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video' 11import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
12import { VideoPathManager } from '@server/lib/video-path-manager' 12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { buildNextVideoState } from '@server/lib/video-state' 13import { buildNextVideoState } from '@server/lib/video-state'
14import { ThumbnailModel } from '@server/models/video/thumbnail' 14import { ThumbnailModel } from '@server/models/video/thumbnail'
@@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils'
18import { 18import {
19 ThumbnailType, 19 ThumbnailType,
20 VideoImportPayload, 20 VideoImportPayload,
21 VideoImportPreventExceptionResult,
21 VideoImportState, 22 VideoImportState,
22 VideoImportTorrentPayload, 23 VideoImportTorrentPayload,
23 VideoImportTorrentPayloadType, 24 VideoImportTorrentPayloadType,
@@ -41,20 +42,29 @@ import { Notifier } from '../../notifier'
41import { generateVideoMiniature } from '../../thumbnail' 42import { generateVideoMiniature } from '../../thumbnail'
42import { JobQueue } from '../job-queue' 43import { JobQueue } from '../job-queue'
43 44
44async function processVideoImport (job: Job) { 45async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
45 const payload = job.data as VideoImportPayload 46 const payload = job.data as VideoImportPayload
46 47
47 const videoImport = await getVideoImportOrDie(payload) 48 const videoImport = await getVideoImportOrDie(payload)
48 if (videoImport.state === VideoImportState.CANCELLED) { 49 if (videoImport.state === VideoImportState.CANCELLED) {
49 logger.info('Do not process import since it has been cancelled', { payload }) 50 logger.info('Do not process import since it has been cancelled', { payload })
50 return 51 return { resultType: 'success' }
51 } 52 }
52 53
53 videoImport.state = VideoImportState.PROCESSING 54 videoImport.state = VideoImportState.PROCESSING
54 await videoImport.save() 55 await videoImport.save()
55 56
56 if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload) 57 try {
57 if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload) 58 if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
59 if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
60
61 return { resultType: 'success' }
62 } catch (err) {
63 if (!payload.preventException) throw err
64
65 logger.warn('Catch error in video import to send value to parent job.', { payload, err })
66 return { resultType: 'error' }
67 }
58} 68}
59 69
60// --------------------------------------------------------------------------- 70// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 281e2e51a..3970d48b7 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -22,6 +22,7 @@ import {
22 ActivitypubHttpFetcherPayload, 22 ActivitypubHttpFetcherPayload,
23 ActivitypubHttpUnicastPayload, 23 ActivitypubHttpUnicastPayload,
24 ActorKeysPayload, 24 ActorKeysPayload,
25 AfterVideoChannelImportPayload,
25 DeleteResumableUploadMetaFilePayload, 26 DeleteResumableUploadMetaFilePayload,
26 EmailPayload, 27 EmailPayload,
27 FederateVideoPayload, 28 FederateVideoPayload,
@@ -31,6 +32,7 @@ import {
31 MoveObjectStoragePayload, 32 MoveObjectStoragePayload,
32 NotifyPayload, 33 NotifyPayload,
33 RefreshPayload, 34 RefreshPayload,
35 VideoChannelImportPayload,
34 VideoFileImportPayload, 36 VideoFileImportPayload,
35 VideoImportPayload, 37 VideoImportPayload,
36 VideoLiveEndingPayload, 38 VideoLiveEndingPayload,
@@ -53,12 +55,14 @@ import { processFederateVideo } from './handlers/federate-video'
53import { processManageVideoTorrent } from './handlers/manage-video-torrent' 55import { processManageVideoTorrent } from './handlers/manage-video-torrent'
54import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' 56import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
55import { processNotify } from './handlers/notify' 57import { processNotify } from './handlers/notify'
58import { processVideoChannelImport } from './handlers/video-channel-import'
56import { processVideoFileImport } from './handlers/video-file-import' 59import { processVideoFileImport } from './handlers/video-file-import'
57import { processVideoImport } from './handlers/video-import' 60import { processVideoImport } from './handlers/video-import'
58import { processVideoLiveEnding } from './handlers/video-live-ending' 61import { processVideoLiveEnding } from './handlers/video-live-ending'
59import { processVideoStudioEdition } from './handlers/video-studio-edition' 62import { processVideoStudioEdition } from './handlers/video-studio-edition'
60import { processVideoTranscoding } from './handlers/video-transcoding' 63import { processVideoTranscoding } from './handlers/video-transcoding'
61import { processVideosViewsStats } from './handlers/video-views-stats' 64import { processVideosViewsStats } from './handlers/video-views-stats'
65import { processAfterVideoChannelImport } from './handlers/after-video-channel-import'
62 66
63export type CreateJobArgument = 67export type CreateJobArgument =
64 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 68 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -79,6 +83,9 @@ export type CreateJobArgument =
79 { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | 83 { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
80 { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | 84 { type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
81 { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | 85 { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
86 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
87 { type: 'video-channel-import', payload: VideoChannelImportPayload } |
88 { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
82 { type: 'notify', payload: NotifyPayload } | 89 { type: 'notify', payload: NotifyPayload } |
83 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | 90 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
84 { type: 'federate-video', payload: FederateVideoPayload } 91 { type: 'federate-video', payload: FederateVideoPayload }
@@ -106,8 +113,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
106 'video-redundancy': processVideoRedundancy, 113 'video-redundancy': processVideoRedundancy,
107 'move-to-object-storage': processMoveToObjectStorage, 114 'move-to-object-storage': processMoveToObjectStorage,
108 'manage-video-torrent': processManageVideoTorrent, 115 'manage-video-torrent': processManageVideoTorrent,
109 'notify': processNotify,
110 'video-studio-edition': processVideoStudioEdition, 116 'video-studio-edition': processVideoStudioEdition,
117 'video-channel-import': processVideoChannelImport,
118 'after-video-channel-import': processAfterVideoChannelImport,
119 'notify': processNotify,
111 'federate-video': processFederateVideo 120 'federate-video': processFederateVideo
112} 121}
113 122
@@ -134,6 +143,8 @@ const jobTypes: JobType[] = [
134 'move-to-object-storage', 143 'move-to-object-storage',
135 'manage-video-torrent', 144 'manage-video-torrent',
136 'video-studio-edition', 145 'video-studio-edition',
146 'video-channel-import',
147 'after-video-channel-import',
137 'notify', 148 'notify',
138 'federate-video' 149 'federate-video'
139] 150]
@@ -306,7 +317,7 @@ class JobQueue {
306 .catch(err => logger.error('Cannot create job.', { err, options })) 317 .catch(err => logger.error('Cannot create job.', { err, options }))
307 } 318 }
308 319
309 async createJob (options: CreateJobArgument & CreateJobOptions) { 320 createJob (options: CreateJobArgument & CreateJobOptions) {
310 const queue: Queue = this.queues[options.type] 321 const queue: Queue = this.queues[options.type]
311 if (queue === undefined) { 322 if (queue === undefined) {
312 logger.error('Unknown queue %s: cannot create job.', options.type) 323 logger.error('Unknown queue %s: cannot create job.', options.type)
@@ -318,7 +329,7 @@ class JobQueue {
318 return queue.add('job', options.payload, jobOptions) 329 return queue.add('job', options.payload, jobOptions)
319 } 330 }
320 331
321 async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { 332 createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
322 let lastJob: FlowJob 333 let lastJob: FlowJob
323 334
324 for (const job of jobs) { 335 for (const job of jobs) {
@@ -336,7 +347,7 @@ class JobQueue {
336 return this.flowProducer.add(lastJob) 347 return this.flowProducer.add(lastJob)
337 } 348 }
338 349
339 async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { 350 createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
340 return this.flowProducer.add({ 351 return this.flowProducer.add({
341 ...this.buildJobFlowOption(parent), 352 ...this.buildJobFlowOption(parent),
342 353
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
new file mode 100644
index 000000000..fd9a35299
--- /dev/null
+++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -0,0 +1,61 @@
1import { logger } from '@server/helpers/logger'
2import { CONFIG } from '@server/initializers/config'
3import { VideoChannelModel } from '@server/models/video/video-channel'
4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
5import { VideoChannelSyncState } from '@shared/models'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { synchronizeChannel } from '../sync-channel'
8import { AbstractScheduler } from './abstract-scheduler'
9
10export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
11 private static instance: AbstractScheduler
12 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
13
14 private constructor () {
15 super()
16 }
17
18 protected async internalExecute () {
19 logger.debug('Running %s.%s', this.constructor.name, this.internalExecute.name)
20
21 if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
22 logger.info('Discard channels synchronization as the feature is disabled')
23 return
24 }
25
26 const channelSyncs = await VideoChannelSyncModel.listSyncs()
27
28 for (const sync of channelSyncs) {
29 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
30
31 try {
32 logger.info(
33 'Creating video import jobs for "%s" sync with external channel "%s"',
34 channel.Actor.preferredUsername, sync.externalChannelUrl
35 )
36
37 const onlyAfter = sync.lastSyncAt || sync.createdAt
38
39 sync.state = VideoChannelSyncState.PROCESSING
40 sync.lastSyncAt = new Date()
41 await sync.save()
42
43 await synchronizeChannel({
44 channel,
45 externalChannelUrl: sync.externalChannelUrl,
46 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
47 channelSync: sync,
48 onlyAfter
49 })
50 } catch (err) {
51 logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
52 sync.state = VideoChannelSyncState.FAILED
53 await sync.save()
54 }
55 }
56 }
57
58 static get Instance () {
59 return this.instance || (this.instance = new this())
60 }
61}
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts
index a3312fa20..78a9546ae 100644
--- a/server/lib/server-config-manager.ts
+++ b/server/lib/server-config-manager.ts
@@ -170,6 +170,9 @@ class ServerConfigManager {
170 torrent: { 170 torrent: {
171 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED 171 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
172 } 172 }
173 },
174 videoChannelSynchronization: {
175 enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED
173 } 176 }
174 }, 177 },
175 autoBlacklist: { 178 autoBlacklist: {
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
new file mode 100644
index 000000000..50f80e6f9
--- /dev/null
+++ b/server/lib/sync-channel.ts
@@ -0,0 +1,81 @@
1import { logger } from '@server/helpers/logger'
2import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
3import { CONFIG } from '@server/initializers/config'
4import { buildYoutubeDLImport } from '@server/lib/video-import'
5import { UserModel } from '@server/models/user/user'
6import { VideoImportModel } from '@server/models/video/video-import'
7import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
8import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
9import { CreateJobArgument, JobQueue } from './job-queue'
10import { ServerConfigManager } from './server-config-manager'
11
12export async function synchronizeChannel (options: {
13 channel: MChannelAccountDefault
14 externalChannelUrl: string
15 channelSync?: MChannelSync
16 videosCountLimit?: number
17 onlyAfter?: Date
18}) {
19 const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
20
21 const user = await UserModel.loadByChannelActorId(channel.actorId)
22 const youtubeDL = new YoutubeDLWrapper(
23 externalChannelUrl,
24 ServerConfigManager.Instance.getEnabledResolutions('vod'),
25 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
26 )
27
28 const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
29
30 const targetUrls = infoList
31 .filter(videoInfo => {
32 if (!onlyAfter) return true
33
34 return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
35 })
36 .map(videoInfo => videoInfo.webpageUrl)
37
38 logger.info(
39 'Fetched %d candidate URLs for sync channel %s.',
40 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
41 )
42
43 if (targetUrls.length === 0) {
44 if (channelSync) {
45 channelSync.state = VideoChannelSyncState.SYNCED
46 await channelSync.save()
47 }
48
49 return
50 }
51
52 const children: CreateJobArgument[] = []
53
54 for (const targetUrl of targetUrls) {
55 if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
56 logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
57 continue
58 }
59
60 const { job } = await buildYoutubeDLImport({
61 user,
62 channel,
63 targetUrl,
64 channelSync,
65 importDataOverride: {
66 privacy: VideoPrivacy.PUBLIC
67 }
68 })
69
70 children.push(job)
71 }
72
73 const parent: CreateJobArgument = {
74 type: 'after-video-channel-import',
75 payload: {
76 channelSyncId: channelSync?.id
77 }
78 }
79
80 await JobQueue.Instance.createJobWithChildren(parent, children)
81}
diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts
new file mode 100644
index 000000000..fb9306967
--- /dev/null
+++ b/server/lib/video-import.ts
@@ -0,0 +1,308 @@
1import { remove } from 'fs-extra'
2import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
3import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
4import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
5import { isResolvingToUnicastOnly } from '@server/helpers/dns'
6import { logger } from '@server/helpers/logger'
7import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
8import { CONFIG } from '@server/initializers/config'
9import { sequelizeTypescript } from '@server/initializers/database'
10import { Hooks } from '@server/lib/plugins/hooks'
11import { ServerConfigManager } from '@server/lib/server-config-manager'
12import { setVideoTags } from '@server/lib/video'
13import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
14import { VideoModel } from '@server/models/video/video'
15import { VideoCaptionModel } from '@server/models/video/video-caption'
16import { VideoImportModel } from '@server/models/video/video-import'
17import { FilteredModelAttributes } from '@server/types'
18import {
19 MChannelAccountDefault,
20 MChannelSync,
21 MThumbnail,
22 MUser,
23 MVideoAccountDefault,
24 MVideoCaption,
25 MVideoImportFormattable,
26 MVideoTag,
27 MVideoThumbnail,
28 MVideoWithBlacklistLight
29} from '@server/types/models'
30import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
31import { getLocalVideoActivityPubUrl } from './activitypub/url'
32import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
33
34class YoutubeDlImportError extends Error {
35 code: YoutubeDlImportError.CODE
36 cause?: Error // Property to remove once ES2022 is used
37 constructor ({ message, code }) {
38 super(message)
39 this.code = code
40 }
41
42 static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
43 const ytDlErr = new this({ message: message ?? err.message, code })
44 ytDlErr.cause = err
45 ytDlErr.stack = err.stack // Useless once ES2022 is used
46 return ytDlErr
47 }
48}
49
50namespace YoutubeDlImportError {
51 export enum CODE {
52 FETCH_ERROR,
53 NOT_ONLY_UNICAST_URL
54 }
55}
56
57// ---------------------------------------------------------------------------
58
59async function insertFromImportIntoDB (parameters: {
60 video: MVideoThumbnail
61 thumbnailModel: MThumbnail
62 previewModel: MThumbnail
63 videoChannel: MChannelAccountDefault
64 tags: string[]
65 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
66 user: MUser
67}): Promise<MVideoImportFormattable> {
68 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
69
70 const videoImport = await sequelizeTypescript.transaction(async t => {
71 const sequelizeOptions = { transaction: t }
72
73 // Save video object in database
74 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
75 videoCreated.VideoChannel = videoChannel
76
77 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
78 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
79
80 await autoBlacklistVideoIfNeeded({
81 video: videoCreated,
82 user,
83 notify: false,
84 isRemote: false,
85 isNew: true,
86 transaction: t
87 })
88
89 await setVideoTags({ video: videoCreated, tags, transaction: t })
90
91 // Create video import object in database
92 const videoImport = await VideoImportModel.create(
93 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
94 sequelizeOptions
95 ) as MVideoImportFormattable
96 videoImport.Video = videoCreated
97
98 return videoImport
99 })
100
101 return videoImport
102}
103
104async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
105 channelId: number
106 importData: YoutubeDLInfo
107 importDataOverride?: Partial<VideoImportCreate>
108 importType: 'url' | 'torrent'
109}): Promise<MVideoThumbnail> {
110 let videoData = {
111 name: importDataOverride?.name || importData.name || 'Unknown name',
112 remote: false,
113 category: importDataOverride?.category || importData.category,
114 licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
115 language: importDataOverride?.language || importData.language,
116 commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
117 downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
118 waitTranscoding: importDataOverride?.waitTranscoding || false,
119 state: VideoState.TO_IMPORT,
120 nsfw: importDataOverride?.nsfw || importData.nsfw || false,
121 description: importDataOverride?.description || importData.description,
122 support: importDataOverride?.support || null,
123 privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
124 duration: 0, // duration will be set by the import job
125 channelId,
126 originallyPublishedAt: importDataOverride?.originallyPublishedAt
127 ? new Date(importDataOverride?.originallyPublishedAt)
128 : importData.originallyPublishedAt
129 }
130
131 videoData = await Hooks.wrapObject(
132 videoData,
133 importType === 'url'
134 ? 'filter:api.video.import-url.video-attribute.result'
135 : 'filter:api.video.import-torrent.video-attribute.result'
136 )
137
138 const video = new VideoModel(videoData)
139 video.url = getLocalVideoActivityPubUrl(video)
140
141 return video
142}
143
144async function buildYoutubeDLImport (options: {
145 targetUrl: string
146 channel: MChannelAccountDefault
147 user: MUser
148 channelSync?: MChannelSync
149 importDataOverride?: Partial<VideoImportCreate>
150 thumbnailFilePath?: string
151 previewFilePath?: string
152}) {
153 const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
154
155 const youtubeDL = new YoutubeDLWrapper(
156 targetUrl,
157 ServerConfigManager.Instance.getEnabledResolutions('vod'),
158 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
159 )
160
161 // Get video infos
162 let youtubeDLInfo: YoutubeDLInfo
163 try {
164 youtubeDLInfo = await youtubeDL.getInfoForDownload()
165 } catch (err) {
166 throw YoutubeDlImportError.fromError(
167 err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
168 )
169 }
170
171 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
172 throw new YoutubeDlImportError({
173 message: 'Cannot use non unicast IP as targetUrl.',
174 code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
175 })
176 }
177
178 const video = await buildVideoFromImport({
179 channelId: channel.id,
180 importData: youtubeDLInfo,
181 importDataOverride,
182 importType: 'url'
183 })
184
185 const thumbnailModel = await forgeThumbnail({
186 inputPath: thumbnailFilePath,
187 downloadUrl: youtubeDLInfo.thumbnailUrl,
188 video,
189 type: ThumbnailType.MINIATURE
190 })
191
192 const previewModel = await forgeThumbnail({
193 inputPath: previewFilePath,
194 downloadUrl: youtubeDLInfo.thumbnailUrl,
195 video,
196 type: ThumbnailType.PREVIEW
197 })
198
199 const videoImport = await insertFromImportIntoDB({
200 video,
201 thumbnailModel,
202 previewModel,
203 videoChannel: channel,
204 tags: importDataOverride?.tags || youtubeDLInfo.tags,
205 user,
206 videoImportAttributes: {
207 targetUrl,
208 state: VideoImportState.PENDING,
209 userId: user.id
210 }
211 })
212
213 // Get video subtitles
214 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
215
216 let fileExt = `.${youtubeDLInfo.ext}`
217 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
218
219 const payload: VideoImportPayload = {
220 type: 'youtube-dl' as 'youtube-dl',
221 videoImportId: videoImport.id,
222 fileExt,
223 // If part of a sync process, there is a parent job that will aggregate children results
224 preventException: !!channelSync
225 }
226
227 return {
228 videoImport,
229 job: { type: 'video-import' as 'video-import', payload }
230 }
231}
232
233// ---------------------------------------------------------------------------
234
235export {
236 buildYoutubeDLImport,
237 YoutubeDlImportError,
238 insertFromImportIntoDB,
239 buildVideoFromImport
240}
241
242// ---------------------------------------------------------------------------
243
244async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
245 inputPath?: string
246 downloadUrl?: string
247 video: MVideoThumbnail
248 type: ThumbnailType
249}): Promise<MThumbnail> {
250 if (inputPath) {
251 return updateVideoMiniatureFromExisting({
252 inputPath,
253 video,
254 type,
255 automaticallyGenerated: false
256 })
257 } else if (downloadUrl) {
258 try {
259 return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
260 } catch (err) {
261 logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
262 }
263 }
264 return null
265}
266
267async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
268 try {
269 const subtitles = await youtubeDL.getSubtitles()
270
271 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
272
273 for (const subtitle of subtitles) {
274 if (!await isVTTFileValid(subtitle.path)) {
275 await remove(subtitle.path)
276 continue
277 }
278
279 const videoCaption = new VideoCaptionModel({
280 videoId,
281 language: subtitle.language,
282 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
283 }) as MVideoCaption
284
285 // Move physical file
286 await moveAndProcessCaptionFile(subtitle, videoCaption)
287
288 await sequelizeTypescript.transaction(async t => {
289 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
290 })
291 }
292 } catch (err) {
293 logger.warn('Cannot get video subtitles.', { err })
294 }
295}
296
297async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
298 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
299 const uniqHosts = new Set(hosts)
300
301 for (const h of uniqHosts) {
302 if (await isResolvingToUnicastOnly(h) !== true) {
303 return false
304 }
305 }
306
307 return true
308}
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index 9ce47c5aa..f60103f48 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -66,6 +66,8 @@ const customConfigUpdateValidator = [
66 body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'), 66 body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
67 body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'), 67 body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
68 68
69 body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'),
70
69 body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'), 71 body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'),
70 body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'), 72 body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'),
71 73
@@ -110,6 +112,7 @@ const customConfigUpdateValidator = [
110 if (areValidationErrors(req, res)) return 112 if (areValidationErrors(req, res)) return
111 if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return 113 if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
112 if (!checkInvalidTranscodingConfig(req.body, res)) return 114 if (!checkInvalidTranscodingConfig(req.body, res)) return
115 if (!checkInvalidSynchronizationConfig(req.body, res)) return
113 if (!checkInvalidLiveConfig(req.body, res)) return 116 if (!checkInvalidLiveConfig(req.body, res)) return
114 if (!checkInvalidVideoStudioConfig(req.body, res)) return 117 if (!checkInvalidVideoStudioConfig(req.body, res)) return
115 118
@@ -157,6 +160,14 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express
157 return true 160 return true
158} 161}
159 162
163function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) {
164 if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) {
165 res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' })
166 return false
167 }
168 return true
169}
170
160function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { 171function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
161 if (customConfig.live.enabled === false) return true 172 if (customConfig.live.enabled === false) return true
162 173
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts
index c9978e3b4..0354e3fc6 100644
--- a/server/middlewares/validators/sort.ts
+++ b/server/middlewares/validators/sort.ts
@@ -52,6 +52,7 @@ const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAY
52const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS) 52const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS)
53const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) 53const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
54const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) 54const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
55const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS)
55 56
56const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS) 57const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
57const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS) 58const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)
@@ -84,5 +85,6 @@ export {
84 videoPlaylistsSearchSortValidator, 85 videoPlaylistsSearchSortValidator,
85 accountsFollowersSortValidator, 86 accountsFollowersSortValidator,
86 videoChannelsFollowersSortValidator, 87 videoChannelsFollowersSortValidator,
88 videoChannelSyncsSortValidator,
87 pluginsSortValidator 89 pluginsSortValidator
88} 90}
diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts
index 1dd7b5d2e..d225dfe45 100644
--- a/server/middlewares/validators/videos/index.ts
+++ b/server/middlewares/validators/videos/index.ts
@@ -14,3 +14,4 @@ export * from './video-stats'
14export * from './video-studio' 14export * from './video-studio'
15export * from './video-transcoding' 15export * from './video-transcoding'
16export * from './videos' 16export * from './videos'
17export * from './video-channel-sync'
diff --git a/server/middlewares/validators/videos/video-channel-sync.ts b/server/middlewares/validators/videos/video-channel-sync.ts
new file mode 100644
index 000000000..b18498243
--- /dev/null
+++ b/server/middlewares/validators/videos/video-channel-sync.ts
@@ -0,0 +1,66 @@
1import * as express from 'express'
2import { body, param } from 'express-validator'
3import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
4import { logger } from '@server/helpers/logger'
5import { CONFIG } from '@server/initializers/config'
6import { VideoChannelModel } from '@server/models/video/video-channel'
7import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
8import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
9import { areValidationErrors, doesVideoChannelIdExist } from '../shared'
10
11export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => {
12 if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
13 return res.fail({
14 status: HttpStatusCode.FORBIDDEN_403,
15 message: 'Synchronization is impossible as video channel synchronization is not enabled on the server'
16 })
17 }
18
19 return next()
20}
21
22export const videoChannelSyncValidator = [
23 body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
24 body('videoChannelId').isInt().withMessage('Should have a valid video channel id'),
25
26 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
27 logger.debug('Checking videoChannelSync parameters', { parameters: req.body })
28
29 if (areValidationErrors(req, res)) return
30
31 const body: VideoChannelSyncCreate = req.body
32 if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return
33
34 const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId)
35 if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) {
36 return res.fail({
37 message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations`
38 })
39 }
40
41 return next()
42 }
43]
44
45export const ensureSyncExists = [
46 param('id').exists().isInt().withMessage('Should have an sync id'),
47
48 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
49 if (areValidationErrors(req, res)) return
50
51 const syncId = parseInt(req.params.id, 10)
52 const sync = await VideoChannelSyncModel.loadWithChannel(syncId)
53
54 if (!sync) {
55 return res.fail({
56 status: HttpStatusCode.NOT_FOUND_404,
57 message: 'Synchronization not found'
58 })
59 }
60
61 res.locals.videoChannelSync = sync
62 res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
63
64 return next()
65 }
66]
diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts
index 3bfdebbb1..88f8b814d 100644
--- a/server/middlewares/validators/videos/video-channels.ts
+++ b/server/middlewares/validators/videos/video-channels.ts
@@ -1,5 +1,6 @@
1import express from 'express' 1import express from 'express'
2import { body, param, query } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
3import { CONFIG } from '@server/initializers/config' 4import { CONFIG } from '@server/initializers/config'
4import { MChannelAccountDefault } from '@server/types/models' 5import { MChannelAccountDefault } from '@server/types/models'
5import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' 6import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
@@ -13,9 +14,9 @@ import {
13import { logger } from '../../../helpers/logger' 14import { logger } from '../../../helpers/logger'
14import { ActorModel } from '../../../models/actor/actor' 15import { ActorModel } from '../../../models/actor/actor'
15import { VideoChannelModel } from '../../../models/video/video-channel' 16import { VideoChannelModel } from '../../../models/video/video-channel'
16import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared' 17import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared'
17 18
18const videoChannelsAddValidator = [ 19export const videoChannelsAddValidator = [
19 body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'), 20 body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'),
20 body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'), 21 body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'),
21 body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'), 22 body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'),
@@ -45,7 +46,7 @@ const videoChannelsAddValidator = [
45 } 46 }
46] 47]
47 48
48const videoChannelsUpdateValidator = [ 49export const videoChannelsUpdateValidator = [
49 param('nameWithHost').exists().withMessage('Should have an video channel name with host'), 50 param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
50 body('displayName') 51 body('displayName')
51 .optional() 52 .optional()
@@ -69,7 +70,7 @@ const videoChannelsUpdateValidator = [
69 } 70 }
70] 71]
71 72
72const videoChannelsRemoveValidator = [ 73export const videoChannelsRemoveValidator = [
73 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 74 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
74 logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params }) 75 logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params })
75 76
@@ -79,7 +80,7 @@ const videoChannelsRemoveValidator = [
79 } 80 }
80] 81]
81 82
82const videoChannelsNameWithHostValidator = [ 83export const videoChannelsNameWithHostValidator = [
83 param('nameWithHost').exists().withMessage('Should have an video channel name with host'), 84 param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
84 85
85 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 86 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -93,7 +94,7 @@ const videoChannelsNameWithHostValidator = [
93 } 94 }
94] 95]
95 96
96const ensureIsLocalChannel = [ 97export const ensureIsLocalChannel = [
97 (req: express.Request, res: express.Response, next: express.NextFunction) => { 98 (req: express.Request, res: express.Response, next: express.NextFunction) => {
98 if (res.locals.videoChannel.Actor.isOwned() === false) { 99 if (res.locals.videoChannel.Actor.isOwned() === false) {
99 return res.fail({ 100 return res.fail({
@@ -106,7 +107,18 @@ const ensureIsLocalChannel = [
106 } 107 }
107] 108]
108 109
109const videoChannelStatsValidator = [ 110export const ensureChannelOwnerCanUpload = [
111 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
112 const channel = res.locals.videoChannel
113 const user = { id: channel.Account.userId }
114
115 if (!await checkUserQuota(user, 1, res)) return
116
117 next()
118 }
119]
120
121export const videoChannelStatsValidator = [
110 query('withStats') 122 query('withStats')
111 .optional() 123 .optional()
112 .customSanitizer(toBooleanOrNull) 124 .customSanitizer(toBooleanOrNull)
@@ -118,7 +130,7 @@ const videoChannelStatsValidator = [
118 } 130 }
119] 131]
120 132
121const videoChannelsListValidator = [ 133export const videoChannelsListValidator = [
122 query('search').optional().not().isEmpty().withMessage('Should have a valid search'), 134 query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
123 135
124 (req: express.Request, res: express.Response, next: express.NextFunction) => { 136 (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -130,17 +142,24 @@ const videoChannelsListValidator = [
130 } 142 }
131] 143]
132 144
133// --------------------------------------------------------------------------- 145export const videoChannelImportVideosValidator = [
146 body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
134 147
135export { 148 (req: express.Request, res: express.Response, next: express.NextFunction) => {
136 videoChannelsAddValidator, 149 logger.debug('Checking videoChannelImport parameters', { parameters: req.body })
137 videoChannelsUpdateValidator, 150
138 videoChannelsRemoveValidator, 151 if (areValidationErrors(req, res)) return
139 videoChannelsNameWithHostValidator, 152
140 ensureIsLocalChannel, 153 if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
141 videoChannelsListValidator, 154 return res.fail({
142 videoChannelStatsValidator 155 status: HttpStatusCode.FORBIDDEN_403,
143} 156 message: 'Channel import is impossible as video upload via HTTP is not enabled on the server'
157 })
158 }
159
160 return next()
161 }
162]
144 163
145// --------------------------------------------------------------------------- 164// ---------------------------------------------------------------------------
146 165
diff --git a/server/models/utils.ts b/server/models/utils.ts
index c468f748d..1e168d419 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A
117 return getSort(value, lastSort) 117 return getSort(value, lastSort)
118} 118}
119 119
120function getChannelSyncSort (value: string): OrderItem[] {
121 const { direction, field } = buildDirectionAndField(value)
122 if (field.toLowerCase() === 'videochannel') {
123 return [
124 [ literal('"VideoChannel.name"'), direction ]
125 ]
126 }
127 return [ [ field, direction ] ]
128}
129
120function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { 130function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
121 if (!model.createdAt || !model.updatedAt) { 131 if (!model.createdAt || !model.updatedAt) {
122 throw new Error('Miss createdAt & updatedAt attributes to model') 132 throw new Error('Miss createdAt & updatedAt attributes to model')
@@ -280,6 +290,7 @@ export {
280 getAdminUsersSort, 290 getAdminUsersSort,
281 getVideoSort, 291 getVideoSort,
282 getBlacklistSort, 292 getBlacklistSort,
293 getChannelSyncSort,
283 createSimilarityAttribute, 294 createSimilarityAttribute,
284 throwIfNotValid, 295 throwIfNotValid,
285 buildServerIdsFollowedBy, 296 buildServerIdsFollowedBy,
diff --git a/server/models/video/video-channel-sync.ts b/server/models/video/video-channel-sync.ts
new file mode 100644
index 000000000..6e49cde10
--- /dev/null
+++ b/server/models/video/video-channel-sync.ts
@@ -0,0 +1,176 @@
1import { Op } from 'sequelize'
2import {
3 AllowNull,
4 BelongsTo,
5 Column,
6 CreatedAt,
7 DataType,
8 Default,
9 DefaultScope,
10 ForeignKey,
11 Is,
12 Model,
13 Table,
14 UpdatedAt
15} from 'sequelize-typescript'
16import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
17import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs'
18import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
19import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models'
20import { VideoChannelSync, VideoChannelSyncState } from '@shared/models'
21import { AttributesOnly } from '@shared/typescript-utils'
22import { AccountModel } from '../account/account'
23import { UserModel } from '../user/user'
24import { getChannelSyncSort, throwIfNotValid } from '../utils'
25import { VideoChannelModel } from './video-channel'
26
27@DefaultScope(() => ({
28 include: [
29 {
30 model: VideoChannelModel, // Default scope includes avatar and server
31 required: true
32 }
33 ]
34}))
35@Table({
36 tableName: 'videoChannelSync',
37 indexes: [
38 {
39 fields: [ 'videoChannelId' ]
40 }
41 ]
42})
43export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> {
44
45 @AllowNull(false)
46 @Default(null)
47 @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true))
48 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max))
49 externalChannelUrl: string
50
51 @CreatedAt
52 createdAt: Date
53
54 @UpdatedAt
55 updatedAt: Date
56
57 @ForeignKey(() => VideoChannelModel)
58 @Column
59 videoChannelId: number
60
61 @BelongsTo(() => VideoChannelModel, {
62 foreignKey: {
63 allowNull: false
64 },
65 onDelete: 'cascade'
66 })
67 VideoChannel: VideoChannelModel
68
69 @AllowNull(false)
70 @Default(VideoChannelSyncState.WAITING_FIRST_RUN)
71 @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state'))
72 @Column
73 state: VideoChannelSyncState
74
75 @AllowNull(true)
76 @Column(DataType.DATE)
77 lastSyncAt: Date
78
79 static listByAccountForAPI (options: {
80 accountId: number
81 start: number
82 count: number
83 sort: string
84 }) {
85 const getQuery = (forCount: boolean) => {
86 const videoChannelModel = forCount
87 ? VideoChannelModel.unscoped()
88 : VideoChannelModel
89
90 return {
91 offset: options.start,
92 limit: options.count,
93 order: getChannelSyncSort(options.sort),
94 include: [
95 {
96 model: videoChannelModel,
97 required: true,
98 where: {
99 accountId: options.accountId
100 }
101 }
102 ]
103 }
104 }
105
106 return Promise.all([
107 VideoChannelSyncModel.unscoped().count(getQuery(true)),
108 VideoChannelSyncModel.unscoped().findAll(getQuery(false))
109 ]).then(([ total, data ]) => ({ total, data }))
110 }
111
112 static countByAccount (accountId: number) {
113 const query = {
114 include: [
115 {
116 model: VideoChannelModel.unscoped(),
117 required: true,
118 where: {
119 accountId
120 }
121 }
122 ]
123 }
124
125 return VideoChannelSyncModel.unscoped().count(query)
126 }
127
128 static loadWithChannel (id: number): Promise<MChannelSyncChannel> {
129 return VideoChannelSyncModel.findByPk(id)
130 }
131
132 static async listSyncs (): Promise<MChannelSync[]> {
133 const query = {
134 include: [
135 {
136 model: VideoChannelModel.unscoped(),
137 required: true,
138 include: [
139 {
140 model: AccountModel.unscoped(),
141 required: true,
142 include: [ {
143 attributes: [],
144 model: UserModel.unscoped(),
145 required: true,
146 where: {
147 videoQuota: {
148 [Op.ne]: 0
149 },
150 videoQuotaDaily: {
151 [Op.ne]: 0
152 }
153 }
154 } ]
155 }
156 ]
157 }
158 ]
159 }
160 return VideoChannelSyncModel.unscoped().findAll(query)
161 }
162
163 toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync {
164 return {
165 id: this.id,
166 state: {
167 id: this.state,
168 label: VIDEO_CHANNEL_SYNC_STATE[this.state]
169 },
170 externalChannelUrl: this.externalChannelUrl,
171 createdAt: this.createdAt.toISOString(),
172 channel: this.VideoChannel.toFormattedSummaryJSON(),
173 lastSyncAt: this.lastSyncAt?.toISOString()
174 }
175 }
176}
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index 1d8296060..b8e941623 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -1,4 +1,4 @@
1import { WhereOptions } from 'sequelize' 1import { Op, WhereOptions } from 'sequelize'
2import { 2import {
3 AfterUpdate, 3 AfterUpdate,
4 AllowNull, 4 AllowNull,
@@ -161,6 +161,28 @@ export class VideoImportModel extends Model<Partial<AttributesOnly<VideoImportMo
161 ]).then(([ total, data ]) => ({ total, data })) 161 ]).then(([ total, data ]) => ({ total, data }))
162 } 162 }
163 163
164 static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
165 const element = await VideoImportModel.unscoped().findOne({
166 where: {
167 targetUrl,
168 state: {
169 [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
170 }
171 },
172 include: [
173 {
174 model: VideoModel,
175 required: true,
176 where: {
177 channelId
178 }
179 }
180 ]
181 })
182
183 return !!element
184 }
185
164 getTargetIdentifier () { 186 getTargetIdentifier () {
165 return this.targetUrl || this.magnetUri || this.torrentName 187 return this.targetUrl || this.magnetUri || this.torrentName
166 } 188 }
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 2f9f553ab..d67e51123 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -1,7 +1,8 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import 'mocha' 3import 'mocha'
4import { omit } from 'lodash' 4import { merge, omit } from 'lodash'
5import { CustomConfig, HttpStatusCode } from '@shared/models'
5import { 6import {
6 cleanupTests, 7 cleanupTests,
7 createSingleServer, 8 createSingleServer,
@@ -11,7 +12,6 @@ import {
11 PeerTubeServer, 12 PeerTubeServer,
12 setAccessTokensToServers 13 setAccessTokensToServers
13} from '@shared/server-commands' 14} from '@shared/server-commands'
14import { CustomConfig, HttpStatusCode } from '@shared/models'
15 15
16describe('Test config API validators', function () { 16describe('Test config API validators', function () {
17 const path = '/api/v1/config/custom' 17 const path = '/api/v1/config/custom'
@@ -162,6 +162,10 @@ describe('Test config API validators', function () {
162 torrent: { 162 torrent: {
163 enabled: false 163 enabled: false
164 } 164 }
165 },
166 videoChannelSynchronization: {
167 enabled: false,
168 maxPerUser: 10
165 } 169 }
166 }, 170 },
167 trending: { 171 trending: {
@@ -346,7 +350,26 @@ describe('Test config API validators', function () {
346 }) 350 })
347 }) 351 })
348 352
349 it('Should success with the correct parameters', async function () { 353 it('Should fail with a disabled http upload & enabled sync', async function () {
354 const newUpdateParams: CustomConfig = merge({}, updateParams, {
355 import: {
356 videos: {
357 http: { enabled: false }
358 },
359 videoChannelSynchronization: { enabled: true }
360 }
361 })
362
363 await makePutBodyRequest({
364 url: server.url,
365 path,
366 fields: newUpdateParams,
367 token: server.accessToken,
368 expectedStatus: HttpStatusCode.BAD_REQUEST_400
369 })
370 })
371
372 it('Should succeed with the correct parameters', async function () {
350 await makePutBodyRequest({ 373 await makePutBodyRequest({
351 url: server.url, 374 url: server.url,
352 path, 375 path,
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index a27bc8509..5f1168b53 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -27,6 +27,7 @@ import './video-channels'
27import './video-comments' 27import './video-comments'
28import './video-files' 28import './video-files'
29import './video-imports' 29import './video-imports'
30import './video-channel-syncs'
30import './video-playlists' 31import './video-playlists'
31import './video-source' 32import './video-source'
32import './video-studio' 33import './video-studio'
diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts
index deb4a7aa3..f64eafc18 100644
--- a/server/tests/api/check-params/upload-quota.ts
+++ b/server/tests/api/check-params/upload-quota.ts
@@ -70,7 +70,7 @@ describe('Test upload quota', function () {
70 }) 70 })
71 71
72 it('Should fail to import with HTTP/Torrent/magnet', async function () { 72 it('Should fail to import with HTTP/Torrent/magnet', async function () {
73 this.timeout(120000) 73 this.timeout(120_000)
74 74
75 const baseAttributes = { 75 const baseAttributes = {
76 channelId: server.store.channel.id, 76 channelId: server.store.channel.id,
diff --git a/server/tests/api/check-params/video-channel-syncs.ts b/server/tests/api/check-params/video-channel-syncs.ts
new file mode 100644
index 000000000..bcd8984df
--- /dev/null
+++ b/server/tests/api/check-params/video-channel-syncs.ts
@@ -0,0 +1,318 @@
1import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
2import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
3import {
4 ChannelSyncsCommand,
5 createSingleServer,
6 makePostBodyRequest,
7 PeerTubeServer,
8 setAccessTokensToServers,
9 setDefaultVideoChannel
10} from '@shared/server-commands'
11
12describe('Test video channel sync API validator', () => {
13 const path = '/api/v1/video-channel-syncs'
14 let server: PeerTubeServer
15 let command: ChannelSyncsCommand
16 let rootChannelId: number
17 let rootChannelSyncId: number
18 const userInfo = {
19 accessToken: '',
20 username: 'user1',
21 id: -1,
22 channelId: -1,
23 syncId: -1
24 }
25
26 async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> {
27 try {
28 await server.config.disableChannelSync()
29 await callback()
30 } finally {
31 await server.config.enableChannelSync()
32 }
33 }
34
35 async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> {
36 const origConfig = await server.config.getCustomConfig()
37
38 await server.config.updateExistingSubConfig({
39 newConfig: {
40 import: {
41 videoChannelSynchronization: {
42 maxPerUser: maxSync
43 }
44 }
45 }
46 })
47
48 try {
49 await callback()
50 } finally {
51 await server.config.updateCustomConfig({ newCustomConfig: origConfig })
52 }
53 }
54
55 before(async function () {
56 this.timeout(30_000)
57
58 server = await createSingleServer(1)
59
60 await setAccessTokensToServers([ server ])
61 await setDefaultVideoChannel([ server ])
62
63 command = server.channelSyncs
64
65 rootChannelId = server.store.channel.id
66
67 {
68 userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
69
70 const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken })
71 userInfo.id = userId
72 userInfo.channelId = videoChannels[0].id
73 }
74
75 await server.config.enableChannelSync()
76 })
77
78 describe('When creating a sync', function () {
79 let baseCorrectParams: VideoChannelSyncCreate
80
81 before(function () {
82 baseCorrectParams = {
83 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
84 videoChannelId: rootChannelId
85 }
86 })
87
88 it('Should fail when sync is disabled', async function () {
89 await withChannelSyncDisabled(async () => {
90 await command.create({
91 token: server.accessToken,
92 attributes: baseCorrectParams,
93 expectedStatus: HttpStatusCode.FORBIDDEN_403
94 })
95 })
96 })
97
98 it('Should fail with nothing', async function () {
99 const fields = {}
100 await makePostBodyRequest({
101 url: server.url,
102 path,
103 token: server.accessToken,
104 fields,
105 expectedStatus: HttpStatusCode.BAD_REQUEST_400
106 })
107 })
108
109 it('Should fail with no authentication', async function () {
110 await command.create({
111 token: null,
112 attributes: baseCorrectParams,
113 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
114 })
115 })
116
117 it('Should fail without a target url', async function () {
118 const attributes: VideoChannelSyncCreate = {
119 ...baseCorrectParams,
120 externalChannelUrl: null
121 }
122 await command.create({
123 token: server.accessToken,
124 attributes,
125 expectedStatus: HttpStatusCode.BAD_REQUEST_400
126 })
127 })
128
129 it('Should fail without a channelId', async function () {
130 const attributes: VideoChannelSyncCreate = {
131 ...baseCorrectParams,
132 videoChannelId: null
133 }
134 await command.create({
135 token: server.accessToken,
136 attributes,
137 expectedStatus: HttpStatusCode.BAD_REQUEST_400
138 })
139 })
140
141 it('Should fail with a channelId refering nothing', async function () {
142 const attributes: VideoChannelSyncCreate = {
143 ...baseCorrectParams,
144 videoChannelId: 42
145 }
146 await command.create({
147 token: server.accessToken,
148 attributes,
149 expectedStatus: HttpStatusCode.NOT_FOUND_404
150 })
151 })
152
153 it('Should fail to create a sync when the user does not own the channel', async function () {
154 await command.create({
155 token: userInfo.accessToken,
156 attributes: baseCorrectParams,
157 expectedStatus: HttpStatusCode.FORBIDDEN_403
158 })
159 })
160
161 it('Should succeed to create a sync with root and for another user\'s channel', async function () {
162 const { videoChannelSync } = await command.create({
163 token: server.accessToken,
164 attributes: {
165 ...baseCorrectParams,
166 videoChannelId: userInfo.channelId
167 },
168 expectedStatus: HttpStatusCode.OK_200
169 })
170 userInfo.syncId = videoChannelSync.id
171 })
172
173 it('Should succeed with the correct parameters', async function () {
174 const { videoChannelSync } = await command.create({
175 token: server.accessToken,
176 attributes: baseCorrectParams,
177 expectedStatus: HttpStatusCode.OK_200
178 })
179 rootChannelSyncId = videoChannelSync.id
180 })
181
182 it('Should fail when the user exceeds allowed number of synchronizations', async function () {
183 await withMaxSyncsPerUser(1, async () => {
184 await command.create({
185 token: server.accessToken,
186 attributes: {
187 ...baseCorrectParams,
188 videoChannelId: userInfo.channelId
189 },
190 expectedStatus: HttpStatusCode.BAD_REQUEST_400
191 })
192 })
193 })
194 })
195
196 describe('When listing my channel syncs', function () {
197 const myPath = '/api/v1/accounts/root/video-channel-syncs'
198
199 it('Should fail with a bad start pagination', async function () {
200 await checkBadStartPagination(server.url, myPath, server.accessToken)
201 })
202
203 it('Should fail with a bad count pagination', async function () {
204 await checkBadCountPagination(server.url, myPath, server.accessToken)
205 })
206
207 it('Should fail with an incorrect sort', async function () {
208 await checkBadSortPagination(server.url, myPath, server.accessToken)
209 })
210
211 it('Should succeed with the correct parameters', async function () {
212 await command.listByAccount({
213 accountName: 'root',
214 token: server.accessToken,
215 expectedStatus: HttpStatusCode.OK_200
216 })
217 })
218
219 it('Should fail with no authentication', async function () {
220 await command.listByAccount({
221 accountName: 'root',
222 token: null,
223 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
224 })
225 })
226
227 it('Should fail when a simple user lists another user\'s synchronizations', async function () {
228 await command.listByAccount({
229 accountName: 'root',
230 token: userInfo.accessToken,
231 expectedStatus: HttpStatusCode.FORBIDDEN_403
232 })
233 })
234
235 it('Should succeed when root lists another user\'s synchronizations', async function () {
236 await command.listByAccount({
237 accountName: userInfo.username,
238 token: server.accessToken,
239 expectedStatus: HttpStatusCode.OK_200
240 })
241 })
242
243 it('Should succeed even with synchronization disabled', async function () {
244 await withChannelSyncDisabled(async function () {
245 await command.listByAccount({
246 accountName: 'root',
247 token: server.accessToken,
248 expectedStatus: HttpStatusCode.OK_200
249 })
250 })
251 })
252 })
253
254 describe('When triggering deletion', function () {
255 it('should fail with no authentication', async function () {
256 await command.delete({
257 channelSyncId: userInfo.syncId,
258 token: null,
259 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
260 })
261 })
262
263 it('Should fail when channelSyncId does not refer to any sync', async function () {
264 await command.delete({
265 channelSyncId: 42,
266 token: server.accessToken,
267 expectedStatus: HttpStatusCode.NOT_FOUND_404
268 })
269 })
270
271 it('Should fail when sync is not owned by the user', async function () {
272 await command.delete({
273 channelSyncId: rootChannelSyncId,
274 token: userInfo.accessToken,
275 expectedStatus: HttpStatusCode.FORBIDDEN_403
276 })
277 })
278
279 it('Should succeed when root delete a sync they do not own', async function () {
280 await command.delete({
281 channelSyncId: userInfo.syncId,
282 token: server.accessToken,
283 expectedStatus: HttpStatusCode.NO_CONTENT_204
284 })
285 })
286
287 it('should succeed when user delete a sync they own', async function () {
288 const { videoChannelSync } = await command.create({
289 attributes: {
290 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
291 videoChannelId: userInfo.channelId
292 },
293 token: server.accessToken,
294 expectedStatus: HttpStatusCode.OK_200
295 })
296
297 await command.delete({
298 channelSyncId: videoChannelSync.id,
299 token: server.accessToken,
300 expectedStatus: HttpStatusCode.NO_CONTENT_204
301 })
302 })
303
304 it('Should succeed even when synchronization is disabled', async function () {
305 await withChannelSyncDisabled(async function () {
306 await command.delete({
307 channelSyncId: rootChannelSyncId,
308 token: server.accessToken,
309 expectedStatus: HttpStatusCode.NO_CONTENT_204
310 })
311 })
312 })
313 })
314
315 after(async function () {
316 await server?.kill()
317 })
318})
diff --git a/server/tests/api/check-params/video-channels.ts b/server/tests/api/check-params/video-channels.ts
index 5c2650fac..337ea1dd4 100644
--- a/server/tests/api/check-params/video-channels.ts
+++ b/server/tests/api/check-params/video-channels.ts
@@ -3,8 +3,8 @@
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { omit } from 'lodash' 5import { omit } from 'lodash'
6import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' 6import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
7import { buildAbsoluteFixturePath } from '@shared/core-utils' 7import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils'
8import { HttpStatusCode, VideoChannelUpdate } from '@shared/models' 8import { HttpStatusCode, VideoChannelUpdate } from '@shared/models'
9import { 9import {
10 ChannelsCommand, 10 ChannelsCommand,
@@ -23,7 +23,13 @@ const expect = chai.expect
23describe('Test video channels API validator', function () { 23describe('Test video channels API validator', function () {
24 const videoChannelPath = '/api/v1/video-channels' 24 const videoChannelPath = '/api/v1/video-channels'
25 let server: PeerTubeServer 25 let server: PeerTubeServer
26 let accessTokenUser: string 26 const userInfo = {
27 accessToken: '',
28 channelName: 'fake_channel',
29 id: -1,
30 videoQuota: -1,
31 videoQuotaDaily: -1
32 }
27 let command: ChannelsCommand 33 let command: ChannelsCommand
28 34
29 // --------------------------------------------------------------- 35 // ---------------------------------------------------------------
@@ -35,14 +41,15 @@ describe('Test video channels API validator', function () {
35 41
36 await setAccessTokensToServers([ server ]) 42 await setAccessTokensToServers([ server ])
37 43
38 const user = { 44 const userCreds = {
39 username: 'fake', 45 username: 'fake',
40 password: 'fake_password' 46 password: 'fake_password'
41 } 47 }
42 48
43 { 49 {
44 await server.users.create({ username: user.username, password: user.password }) 50 const user = await server.users.create({ username: userCreds.username, password: userCreds.password })
45 accessTokenUser = await server.login.getAccessToken(user) 51 userInfo.id = user.id
52 userInfo.accessToken = await server.login.getAccessToken(userCreds)
46 } 53 }
47 54
48 command = server.channels 55 command = server.channels
@@ -191,7 +198,7 @@ describe('Test video channels API validator', function () {
191 await makePutBodyRequest({ 198 await makePutBodyRequest({
192 url: server.url, 199 url: server.url,
193 path, 200 path,
194 token: accessTokenUser, 201 token: userInfo.accessToken,
195 fields: baseCorrectParams, 202 fields: baseCorrectParams,
196 expectedStatus: HttpStatusCode.FORBIDDEN_403 203 expectedStatus: HttpStatusCode.FORBIDDEN_403
197 }) 204 })
@@ -339,7 +346,7 @@ describe('Test video channels API validator', function () {
339 }) 346 })
340 347
341 it('Should fail with a another user', async function () { 348 it('Should fail with a another user', async function () {
342 await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) 349 await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
343 }) 350 })
344 351
345 it('Should succeed with the correct params', async function () { 352 it('Should succeed with the correct params', async function () {
@@ -347,13 +354,122 @@ describe('Test video channels API validator', function () {
347 }) 354 })
348 }) 355 })
349 356
357 describe('When triggering full synchronization', function () {
358
359 it('Should fail when HTTP upload is disabled', async function () {
360 await server.config.disableImports()
361
362 await command.importVideos({
363 channelName: 'super_channel',
364 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
365 token: server.accessToken,
366 expectedStatus: HttpStatusCode.FORBIDDEN_403
367 })
368
369 await server.config.enableImports()
370 })
371
372 it('Should fail when externalChannelUrl is not provided', async function () {
373 await command.importVideos({
374 channelName: 'super_channel',
375 externalChannelUrl: null,
376 token: server.accessToken,
377 expectedStatus: HttpStatusCode.BAD_REQUEST_400
378 })
379 })
380
381 it('Should fail when externalChannelUrl is malformed', async function () {
382 await command.importVideos({
383 channelName: 'super_channel',
384 externalChannelUrl: 'not-a-url',
385 token: server.accessToken,
386 expectedStatus: HttpStatusCode.BAD_REQUEST_400
387 })
388 })
389
390 it('Should fail with no authentication', async function () {
391 await command.importVideos({
392 channelName: 'super_channel',
393 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
394 token: null,
395 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
396 })
397 })
398
399 it('Should fail when sync is not owned by the user', async function () {
400 await command.importVideos({
401 channelName: 'super_channel',
402 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
403 token: userInfo.accessToken,
404 expectedStatus: HttpStatusCode.FORBIDDEN_403
405 })
406 })
407
408 it('Should fail when the user has no quota', async function () {
409 await server.users.update({
410 userId: userInfo.id,
411 videoQuota: 0
412 })
413
414 await command.importVideos({
415 channelName: 'fake_channel',
416 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
417 token: userInfo.accessToken,
418 expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
419 })
420
421 await server.users.update({
422 userId: userInfo.id,
423 videoQuota: userInfo.videoQuota
424 })
425 })
426
427 it('Should fail when the user has no daily quota', async function () {
428 await server.users.update({
429 userId: userInfo.id,
430 videoQuotaDaily: 0
431 })
432
433 await command.importVideos({
434 channelName: 'fake_channel',
435 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
436 token: userInfo.accessToken,
437 expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
438 })
439
440 await server.users.update({
441 userId: userInfo.id,
442 videoQuotaDaily: userInfo.videoQuotaDaily
443 })
444 })
445
446 it('Should succeed when sync is run by its owner', async function () {
447 if (!areHttpImportTestsDisabled()) return
448
449 await command.importVideos({
450 channelName: 'fake_channel',
451 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
452 token: userInfo.accessToken
453 })
454 })
455
456 it('Should succeed when sync is run with root and for another user\'s channel', async function () {
457 if (!areHttpImportTestsDisabled()) return
458
459 await command.importVideos({
460 channelName: 'fake_channel',
461 externalChannelUrl: FIXTURE_URLS.youtubeChannel
462 })
463 })
464 })
465
350 describe('When deleting a video channel', function () { 466 describe('When deleting a video channel', function () {
351 it('Should fail with a non authenticated user', async function () { 467 it('Should fail with a non authenticated user', async function () {
352 await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) 468 await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
353 }) 469 })
354 470
355 it('Should fail with another authenticated user', async function () { 471 it('Should fail with another authenticated user', async function () {
356 await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) 472 await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
357 }) 473 })
358 474
359 it('Should fail with an unknown video channel id', async function () { 475 it('Should fail with an unknown video channel id', async function () {
diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts
index 4439810e8..5cdd0d925 100644
--- a/server/tests/api/check-params/video-imports.ts
+++ b/server/tests/api/check-params/video-imports.ts
@@ -88,7 +88,13 @@ describe('Test video imports API validator', function () {
88 88
89 it('Should fail with nothing', async function () { 89 it('Should fail with nothing', async function () {
90 const fields = {} 90 const fields = {}
91 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) 91 await makePostBodyRequest({
92 url: server.url,
93 path,
94 token: server.accessToken,
95 fields,
96 expectedStatus: HttpStatusCode.BAD_REQUEST_400
97 })
92 }) 98 })
93 99
94 it('Should fail without a target url', async function () { 100 it('Should fail without a target url', async function () {
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index efc57b345..fc8711161 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -368,6 +368,10 @@ const newCustomConfig: CustomConfig = {
368 torrent: { 368 torrent: {
369 enabled: false 369 enabled: false
370 } 370 }
371 },
372 videoChannelSynchronization: {
373 enabled: false,
374 maxPerUser: 10
371 } 375 }
372 }, 376 },
373 trending: { 377 trending: {
diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts
new file mode 100644
index 000000000..f7540e1ba
--- /dev/null
+++ b/server/tests/api/videos/channel-import-videos.ts
@@ -0,0 +1,50 @@
1import { expect } from 'chai'
2import { FIXTURE_URLS } from '@server/tests/shared'
3import { areHttpImportTestsDisabled } from '@shared/core-utils'
4import {
5 createSingleServer,
6 getServerImportConfig,
7 PeerTubeServer,
8 setAccessTokensToServers,
9 setDefaultVideoChannel,
10 waitJobs
11} from '@shared/server-commands'
12
13describe('Test videos import in a channel', function () {
14 if (areHttpImportTestsDisabled()) return
15
16 function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
17
18 describe('Import using ' + mode, function () {
19 let server: PeerTubeServer
20
21 before(async function () {
22 this.timeout(120_000)
23
24 server = await createSingleServer(1, getServerImportConfig(mode))
25
26 await setAccessTokensToServers([ server ])
27 await setDefaultVideoChannel([ server ])
28
29 await server.config.enableChannelSync()
30 })
31
32 it('Should import a whole channel', async function () {
33 this.timeout(240_000)
34
35 await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel })
36 await waitJobs(server)
37
38 const videos = await server.videos.listByChannel({ handle: server.store.channel.name })
39 expect(videos.total).to.equal(2)
40 })
41
42 after(async function () {
43 await server?.kill()
44 })
45 })
46 }
47
48 runSuite('yt-dlp')
49 runSuite('youtube-dl')
50})
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index a0b6b01cf..266155297 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -4,6 +4,8 @@ import './single-server'
4import './video-captions' 4import './video-captions'
5import './video-change-ownership' 5import './video-change-ownership'
6import './video-channels' 6import './video-channels'
7import './channel-import-videos'
8import './video-channel-syncs'
7import './video-comments' 9import './video-comments'
8import './video-description' 10import './video-description'
9import './video-files' 11import './video-files'
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts
new file mode 100644
index 000000000..229c01f68
--- /dev/null
+++ b/server/tests/api/videos/video-channel-syncs.ts
@@ -0,0 +1,226 @@
1import 'mocha'
2import { expect } from 'chai'
3import { FIXTURE_URLS } from '@server/tests/shared'
4import { areHttpImportTestsDisabled } from '@shared/core-utils'
5import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
6import {
7 ChannelSyncsCommand,
8 createSingleServer,
9 getServerImportConfig,
10 PeerTubeServer,
11 setAccessTokensToServers,
12 setDefaultAccountAvatar,
13 setDefaultChannelAvatar,
14 setDefaultVideoChannel,
15 waitJobs
16} from '@shared/server-commands'
17
18describe('Test channel synchronizations', function () {
19 if (areHttpImportTestsDisabled()) return
20
21 function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
22
23 describe('Sync using ' + mode, function () {
24 let server: PeerTubeServer
25 let command: ChannelSyncsCommand
26 let startTestDate: Date
27 const userInfo = {
28 accessToken: '',
29 username: 'user1',
30 channelName: 'user1_channel',
31 channelId: -1,
32 syncId: -1
33 }
34
35 async function changeDateForSync (channelSyncId: number, newDate: string) {
36 await server.sql.updateQuery(
37 `UPDATE "videoChannelSync" ` +
38 `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
39 `WHERE id=${channelSyncId}`
40 )
41 }
42
43 before(async function () {
44 this.timeout(120_000)
45
46 startTestDate = new Date()
47
48 server = await createSingleServer(1, getServerImportConfig(mode))
49
50 await setAccessTokensToServers([ server ])
51 await setDefaultVideoChannel([ server ])
52 await setDefaultChannelAvatar([ server ])
53 await setDefaultAccountAvatar([ server ])
54
55 await server.config.enableChannelSync()
56
57 command = server.channelSyncs
58
59 {
60 userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
61
62 const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
63 userInfo.channelId = videoChannels[0].id
64 }
65 })
66
67 it('Should fetch the latest channel videos of a remote channel', async function () {
68 this.timeout(120_000)
69
70 {
71 const { video } = await server.imports.importVideo({
72 attributes: {
73 channelId: server.store.channel.id,
74 privacy: VideoPrivacy.PUBLIC,
75 targetUrl: FIXTURE_URLS.youtube
76 }
77 })
78
79 expect(video.name).to.equal('small video - youtube')
80
81 const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
82 expect(total).to.equal(1)
83 }
84
85 const { videoChannelSync } = await command.create({
86 attributes: {
87 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
88 videoChannelId: server.store.channel.id
89 },
90 token: server.accessToken,
91 expectedStatus: HttpStatusCode.OK_200
92 })
93
94 // Ensure any missing video not already fetched will be considered as new
95 await changeDateForSync(videoChannelSync.id, '1970-01-01')
96
97 await server.debug.sendCommand({
98 body: {
99 command: 'process-video-channel-sync-latest'
100 }
101 })
102
103 {
104 await waitJobs(server)
105
106 const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
107 expect(total).to.equal(2)
108 expect(data[0].name).to.equal('test')
109 }
110 })
111
112 it('Should add another synchronization', async function () {
113 const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
114
115 const { videoChannelSync } = await command.create({
116 attributes: {
117 externalChannelUrl,
118 videoChannelId: server.store.channel.id
119 },
120 token: server.accessToken,
121 expectedStatus: HttpStatusCode.OK_200
122 })
123
124 expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
125 expect(videoChannelSync.channel).to.include({
126 id: server.store.channel.id,
127 name: 'root_channel'
128 })
129 expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
130 expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
131 })
132
133 it('Should add a synchronization for another user', async function () {
134 const { videoChannelSync } = await command.create({
135 attributes: {
136 externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
137 videoChannelId: userInfo.channelId
138 },
139 token: userInfo.accessToken
140 })
141 userInfo.syncId = videoChannelSync.id
142 })
143
144 it('Should not import a channel if not asked', async function () {
145 await waitJobs(server)
146
147 const { data } = await command.listByAccount({ accountName: userInfo.username })
148
149 expect(data[0].state).to.contain({
150 id: VideoChannelSyncState.WAITING_FIRST_RUN,
151 label: 'Waiting first run'
152 })
153 })
154
155 it('Should only fetch the videos newer than the creation date', async function () {
156 this.timeout(120_000)
157
158 await changeDateForSync(userInfo.syncId, '2019-03-01')
159
160 await server.debug.sendCommand({
161 body: {
162 command: 'process-video-channel-sync-latest'
163 }
164 })
165
166 await waitJobs(server)
167
168 const { data, total } = await server.videos.listByChannel({
169 handle: userInfo.channelName,
170 include: VideoInclude.NOT_PUBLISHED_STATE
171 })
172
173 expect(total).to.equal(1)
174 expect(data[0].name).to.equal('test')
175 })
176
177 it('Should list channel synchronizations', async function () {
178 // Root
179 {
180 const { total, data } = await command.listByAccount({ accountName: 'root' })
181 expect(total).to.equal(2)
182
183 expect(data[0]).to.deep.contain({
184 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
185 state: {
186 id: VideoChannelSyncState.SYNCED,
187 label: 'Synchronized'
188 }
189 })
190
191 expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
192
193 expect(data[0].channel).to.contain({ id: server.store.channel.id })
194 expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
195 }
196
197 // User
198 {
199 const { total, data } = await command.listByAccount({ accountName: userInfo.username })
200 expect(total).to.equal(1)
201 expect(data[0]).to.deep.contain({
202 externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
203 state: {
204 id: VideoChannelSyncState.SYNCED,
205 label: 'Synchronized'
206 }
207 })
208 }
209 })
210
211 it('Should remove user\'s channel synchronizations', async function () {
212 await command.delete({ channelSyncId: userInfo.syncId })
213
214 const { total } = await command.listByAccount({ accountName: userInfo.username })
215 expect(total).to.equal(0)
216 })
217
218 after(async function () {
219 await server?.kill()
220 })
221 })
222 }
223
224 runSuite('youtube-dl')
225 runSuite('yt-dlp')
226})
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
index 603e2d234..a487062a2 100644
--- a/server/tests/api/videos/video-imports.ts
+++ b/server/tests/api/videos/video-imports.ts
@@ -12,6 +12,7 @@ import {
12 createMultipleServers, 12 createMultipleServers,
13 createSingleServer, 13 createSingleServer,
14 doubleFollow, 14 doubleFollow,
15 getServerImportConfig,
15 PeerTubeServer, 16 PeerTubeServer,
16 setAccessTokensToServers, 17 setAccessTokensToServers,
17 setDefaultVideoChannel, 18 setDefaultVideoChannel,
@@ -84,24 +85,9 @@ describe('Test video imports', function () {
84 let servers: PeerTubeServer[] = [] 85 let servers: PeerTubeServer[] = []
85 86
86 before(async function () { 87 before(async function () {
87 this.timeout(30_000) 88 this.timeout(60_000)
88 89
89 // Run servers 90 servers = await createMultipleServers(2, getServerImportConfig(mode))
90 servers = await createMultipleServers(2, {
91 import: {
92 videos: {
93 http: {
94 youtube_dl_release: {
95 url: mode === 'youtube-dl'
96 ? 'https://yt-dl.org/downloads/latest/youtube-dl'
97 : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
98
99 name: mode
100 }
101 }
102 }
103 }
104 })
105 91
106 await setAccessTokensToServers(servers) 92 await setAccessTokensToServers(servers)
107 await setDefaultVideoChannel(servers) 93 await setDefaultVideoChannel(servers)
diff --git a/server/tests/shared/tests.ts b/server/tests/shared/tests.ts
index 3abaf833d..e67a294dc 100644
--- a/server/tests/shared/tests.ts
+++ b/server/tests/shared/tests.ts
@@ -16,6 +16,8 @@ const FIXTURE_URLS = {
16 */ 16 */
17 youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4', 17 youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4',
18 18
19 youtubeChannel: 'https://youtube.com/channel/UCtnlZdXv3-xQzxiqfn6cjIA',
20
19 // eslint-disable-next-line max-len 21 // eslint-disable-next-line max-len
20 magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4', 22 magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4',
21 23
diff --git a/server/types/express.d.ts b/server/types/express.d.ts
index 8f8c65102..27d60da72 100644
--- a/server/types/express.d.ts
+++ b/server/types/express.d.ts
@@ -8,6 +8,7 @@ import {
8 MActorFollowActorsDefault, 8 MActorFollowActorsDefault,
9 MActorUrl, 9 MActorUrl,
10 MChannelBannerAccountDefault, 10 MChannelBannerAccountDefault,
11 MChannelSyncChannel,
11 MStreamingPlaylist, 12 MStreamingPlaylist,
12 MVideoChangeOwnershipFull, 13 MVideoChangeOwnershipFull,
13 MVideoFile, 14 MVideoFile,
@@ -145,6 +146,7 @@ declare module 'express' {
145 videoStreamingPlaylist?: MStreamingPlaylist 146 videoStreamingPlaylist?: MStreamingPlaylist
146 147
147 videoChannel?: MChannelBannerAccountDefault 148 videoChannel?: MChannelBannerAccountDefault
149 videoChannelSync?: MChannelSyncChannel
148 150
149 videoPlaylistFull?: MVideoPlaylistFull 151 videoPlaylistFull?: MVideoPlaylistFull
150 videoPlaylistSummary?: MVideoPlaylistFullSummary 152 videoPlaylistSummary?: MVideoPlaylistFullSummary
@@ -194,6 +196,7 @@ declare module 'express' {
194 plugin?: MPlugin 196 plugin?: MPlugin
195 197
196 localViewerFull?: MLocalVideoViewerWithWatchSections 198 localViewerFull?: MLocalVideoViewerWithWatchSections
199
197 } 200 }
198 } 201 }
199} 202}
diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts
index fdf8e1ddb..940f0ac0d 100644
--- a/server/types/models/video/index.ts
+++ b/server/types/models/video/index.ts
@@ -8,6 +8,7 @@ export * from './video'
8export * from './video-blacklist' 8export * from './video-blacklist'
9export * from './video-caption' 9export * from './video-caption'
10export * from './video-change-ownership' 10export * from './video-change-ownership'
11export * from './video-channel-sync'
11export * from './video-channels' 12export * from './video-channels'
12export * from './video-comment' 13export * from './video-comment'
13export * from './video-file' 14export * from './video-file'
diff --git a/server/types/models/video/video-channel-sync.ts b/server/types/models/video/video-channel-sync.ts
new file mode 100644
index 000000000..429ab70b0
--- /dev/null
+++ b/server/types/models/video/video-channel-sync.ts
@@ -0,0 +1,17 @@
1import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
2import { FunctionProperties, PickWith } from '@shared/typescript-utils'
3import { MChannelAccountDefault, MChannelFormattable } from './video-channels'
4
5type Use<K extends keyof VideoChannelSyncModel, M> = PickWith<VideoChannelSyncModel, K, M>
6
7export type MChannelSync = Omit<VideoChannelSyncModel, 'VideoChannel'>
8
9export type MChannelSyncChannel =
10 MChannelSync &
11 Use<'VideoChannel', MChannelAccountDefault> &
12 FunctionProperties<VideoChannelSyncModel>
13
14export type MChannelSyncFormattable =
15 FunctionProperties<MChannelSyncChannel> &
16 Use<'VideoChannel', MChannelFormattable> &
17 MChannelSync