From 2a491182e483b97afb1b65c908b23cb48d591807 Mon Sep 17 00:00:00 2001 From: Florent Date: Wed, 10 Aug 2022 09:53:39 +0200 Subject: Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz --- server/middlewares/validators/config.ts | 11 ++++ server/middlewares/validators/sort.ts | 2 + server/middlewares/validators/videos/index.ts | 1 + .../validators/videos/video-channel-sync.ts | 66 ++++++++++++++++++++++ .../validators/videos/video-channels.ts | 55 ++++++++++++------ 5 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 server/middlewares/validators/videos/video-channel-sync.ts (limited to 'server/middlewares') diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index 9ce47c5aa..f60103f48 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -66,6 +66,8 @@ const customConfigUpdateValidator = [ body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'), body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'), + body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'), + body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'), body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'), @@ -110,6 +112,7 @@ const customConfigUpdateValidator = [ if (areValidationErrors(req, res)) return if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return if (!checkInvalidTranscodingConfig(req.body, res)) return + if (!checkInvalidSynchronizationConfig(req.body, res)) return if (!checkInvalidLiveConfig(req.body, res)) return if (!checkInvalidVideoStudioConfig(req.body, res)) return @@ -157,6 +160,14 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express return true } +function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) { + if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) { + res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' }) + return false + } + return true +} + function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { if (customConfig.live.enabled === false) return true diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index c9978e3b4..0354e3fc6 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts @@ -52,6 +52,7 @@ const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAY const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS) const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) +const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS) const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS) const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS) @@ -84,5 +85,6 @@ export { videoPlaylistsSearchSortValidator, accountsFollowersSortValidator, videoChannelsFollowersSortValidator, + videoChannelSyncsSortValidator, pluginsSortValidator } diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts index 1dd7b5d2e..d225dfe45 100644 --- a/server/middlewares/validators/videos/index.ts +++ b/server/middlewares/validators/videos/index.ts @@ -14,3 +14,4 @@ export * from './video-stats' export * from './video-studio' export * from './video-transcoding' export * from './videos' +export * from './video-channel-sync' diff --git a/server/middlewares/validators/videos/video-channel-sync.ts b/server/middlewares/validators/videos/video-channel-sync.ts new file mode 100644 index 000000000..b18498243 --- /dev/null +++ b/server/middlewares/validators/videos/video-channel-sync.ts @@ -0,0 +1,66 @@ +import * as express from 'express' +import { body, param } from 'express-validator' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' +import { logger } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { VideoChannelModel } from '@server/models/video/video-channel' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' +import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models' +import { areValidationErrors, doesVideoChannelIdExist } from '../shared' + +export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Synchronization is impossible as video channel synchronization is not enabled on the server' + }) + } + + return next() +} + +export const videoChannelSyncValidator = [ + body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'), + body('videoChannelId').isInt().withMessage('Should have a valid video channel id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoChannelSync parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + + const body: VideoChannelSyncCreate = req.body + if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return + + const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId) + if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) { + return res.fail({ + message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations` + }) + } + + return next() + } +] + +export const ensureSyncExists = [ + param('id').exists().isInt().withMessage('Should have an sync id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const syncId = parseInt(req.params.id, 10) + const sync = await VideoChannelSyncModel.loadWithChannel(syncId) + + if (!sync) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Synchronization not found' + }) + } + + res.locals.videoChannelSync = sync + res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) + + return next() + } +] diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts index 3bfdebbb1..88f8b814d 100644 --- a/server/middlewares/validators/videos/video-channels.ts +++ b/server/middlewares/validators/videos/video-channels.ts @@ -1,5 +1,6 @@ import express from 'express' import { body, param, query } from 'express-validator' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' import { CONFIG } from '@server/initializers/config' import { MChannelAccountDefault } from '@server/types/models' import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' @@ -13,9 +14,9 @@ import { import { logger } from '../../../helpers/logger' import { ActorModel } from '../../../models/actor/actor' import { VideoChannelModel } from '../../../models/video/video-channel' -import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared' +import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared' -const videoChannelsAddValidator = [ +export const videoChannelsAddValidator = [ body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'), body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'), body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'), @@ -45,7 +46,7 @@ const videoChannelsAddValidator = [ } ] -const videoChannelsUpdateValidator = [ +export const videoChannelsUpdateValidator = [ param('nameWithHost').exists().withMessage('Should have an video channel name with host'), body('displayName') .optional() @@ -69,7 +70,7 @@ const videoChannelsUpdateValidator = [ } ] -const videoChannelsRemoveValidator = [ +export const videoChannelsRemoveValidator = [ async (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params }) @@ -79,7 +80,7 @@ const videoChannelsRemoveValidator = [ } ] -const videoChannelsNameWithHostValidator = [ +export const videoChannelsNameWithHostValidator = [ param('nameWithHost').exists().withMessage('Should have an video channel name with host'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { @@ -93,7 +94,7 @@ const videoChannelsNameWithHostValidator = [ } ] -const ensureIsLocalChannel = [ +export const ensureIsLocalChannel = [ (req: express.Request, res: express.Response, next: express.NextFunction) => { if (res.locals.videoChannel.Actor.isOwned() === false) { return res.fail({ @@ -106,7 +107,18 @@ const ensureIsLocalChannel = [ } ] -const videoChannelStatsValidator = [ +export const ensureChannelOwnerCanUpload = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const channel = res.locals.videoChannel + const user = { id: channel.Account.userId } + + if (!await checkUserQuota(user, 1, res)) return + + next() + } +] + +export const videoChannelStatsValidator = [ query('withStats') .optional() .customSanitizer(toBooleanOrNull) @@ -118,7 +130,7 @@ const videoChannelStatsValidator = [ } ] -const videoChannelsListValidator = [ +export const videoChannelsListValidator = [ query('search').optional().not().isEmpty().withMessage('Should have a valid search'), (req: express.Request, res: express.Response, next: express.NextFunction) => { @@ -130,17 +142,24 @@ const videoChannelsListValidator = [ } ] -// --------------------------------------------------------------------------- +export const videoChannelImportVideosValidator = [ + body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'), -export { - videoChannelsAddValidator, - videoChannelsUpdateValidator, - videoChannelsRemoveValidator, - videoChannelsNameWithHostValidator, - ensureIsLocalChannel, - videoChannelsListValidator, - videoChannelStatsValidator -} + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoChannelImport parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + + if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Channel import is impossible as video upload via HTTP is not enabled on the server' + }) + } + + return next() + } +] // --------------------------------------------------------------------------- -- cgit v1.2.3