Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

5 files changed, 117 insertions, 18 deletions

diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index 9ce47c5aa..f60103f48 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -66,6 +66,8 @@ const customConfigUpdateValidator = [
   body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
   body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
 
+  body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'),
+
   body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'),
   body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'),
 
@@ -110,6 +112,7 @@ const customConfigUpdateValidator = [
     if (areValidationErrors(req, res)) return
     if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
     if (!checkInvalidTranscodingConfig(req.body, res)) return
+    if (!checkInvalidSynchronizationConfig(req.body, res)) return
     if (!checkInvalidLiveConfig(req.body, res)) return
     if (!checkInvalidVideoStudioConfig(req.body, res)) return
 
@@ -157,6 +160,14 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express
   return true
 }
 
+function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) {
+  if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) {
+    res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' })
+    return false
+  }
+  return true
+}
+
 function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
   if (customConfig.live.enabled === false) return true
 
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts
index c9978e3b4..0354e3fc6 100644
--- a/server/middlewares/validators/sort.ts
+++ b/server/middlewares/validators/sort.ts
@@ -52,6 +52,7 @@ const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAY
 const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS)
 const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
 const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
+const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS)
 
 const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
 const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)
@@ -84,5 +85,6 @@ export {
   videoPlaylistsSearchSortValidator,
   accountsFollowersSortValidator,
   videoChannelsFollowersSortValidator,
+  videoChannelSyncsSortValidator,
   pluginsSortValidator
 }
diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts
index 1dd7b5d2e..d225dfe45 100644
--- a/server/middlewares/validators/videos/index.ts
+++ b/server/middlewares/validators/videos/index.ts
@@ -14,3 +14,4 @@ export * from './video-stats'
 export * from './video-studio'
 export * from './video-transcoding'
 export * from './videos'
+export * from './video-channel-sync'
diff --git a/server/middlewares/validators/videos/video-channel-sync.ts b/server/middlewares/validators/videos/video-channel-sync.ts
new file mode 100644
index 000000000..b18498243
--- /dev/null
+++ b/server/middlewares/validators/videos/video-channel-sync.ts
@@ -0,0 +1,66 @@
+import * as express from 'express'
+import { body, param } from 'express-validator'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
+import { areValidationErrors, doesVideoChannelIdExist } from '../shared'
+
+export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => {
+  if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+    return res.fail({
+      status: HttpStatusCode.FORBIDDEN_403,
+      message: 'Synchronization is impossible as video channel synchronization is not enabled on the server'
+    })
+  }
+
+  return next()
+}
+
+export const videoChannelSyncValidator = [
+  body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
+  body('videoChannelId').isInt().withMessage('Should have a valid video channel id'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking videoChannelSync parameters', { parameters: req.body })
+
+    if (areValidationErrors(req, res)) return
+
+    const body: VideoChannelSyncCreate = req.body
+    if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return
+
+    const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId)
+    if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) {
+      return res.fail({
+        message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations`
+      })
+    }
+
+    return next()
+  }
+]
+
+export const ensureSyncExists = [
+  param('id').exists().isInt().withMessage('Should have an sync id'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res)) return
+
+    const syncId = parseInt(req.params.id, 10)
+    const sync = await VideoChannelSyncModel.loadWithChannel(syncId)
+
+    if (!sync) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Synchronization not found'
+      })
+    }
+
+    res.locals.videoChannelSync = sync
+    res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
+
+    return next()
+  }
+]
diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts
index 3bfdebbb1..88f8b814d 100644
--- a/server/middlewares/validators/videos/video-channels.ts
+++ b/server/middlewares/validators/videos/video-channels.ts
@@ -1,5 +1,6 @@
 import express from 'express'
 import { body, param, query } from 'express-validator'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
 import { CONFIG } from '@server/initializers/config'
 import { MChannelAccountDefault } from '@server/types/models'
 import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
@@ -13,9 +14,9 @@ import {
 import { logger } from '../../../helpers/logger'
 import { ActorModel } from '../../../models/actor/actor'
 import { VideoChannelModel } from '../../../models/video/video-channel'
-import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared'
+import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared'
 
-const videoChannelsAddValidator = [
+export const videoChannelsAddValidator = [
   body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'),
   body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'),
   body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'),
@@ -45,7 +46,7 @@ const videoChannelsAddValidator = [
   }
 ]
 
-const videoChannelsUpdateValidator = [
+export const videoChannelsUpdateValidator = [
   param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
   body('displayName')
     .optional()
@@ -69,7 +70,7 @@ const videoChannelsUpdateValidator = [
   }
 ]
 
-const videoChannelsRemoveValidator = [
+export const videoChannelsRemoveValidator = [
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
     logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params })
 
@@ -79,7 +80,7 @@ const videoChannelsRemoveValidator = [
   }
 ]
 
-const videoChannelsNameWithHostValidator = [
+export const videoChannelsNameWithHostValidator = [
   param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -93,7 +94,7 @@ const videoChannelsNameWithHostValidator = [
   }
 ]
 
-const ensureIsLocalChannel = [
+export const ensureIsLocalChannel = [
   (req: express.Request, res: express.Response, next: express.NextFunction) => {
     if (res.locals.videoChannel.Actor.isOwned() === false) {
       return res.fail({
@@ -106,7 +107,18 @@ const ensureIsLocalChannel = [
   }
 ]
 
-const videoChannelStatsValidator = [
+export const ensureChannelOwnerCanUpload = [
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    const channel = res.locals.videoChannel
+    const user = { id: channel.Account.userId }
+
+    if (!await checkUserQuota(user, 1, res)) return
+
+    next()
+  }
+]
+
+export const videoChannelStatsValidator = [
   query('withStats')
     .optional()
     .customSanitizer(toBooleanOrNull)
@@ -118,7 +130,7 @@ const videoChannelStatsValidator = [
   }
 ]
 
-const videoChannelsListValidator = [
+export const videoChannelsListValidator = [
   query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
 
   (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -130,17 +142,24 @@ const videoChannelsListValidator = [
   }
 ]
 
-// ---------------------------------------------------------------------------
+export const videoChannelImportVideosValidator = [
+  body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
 
-export {
-  videoChannelsAddValidator,
-  videoChannelsUpdateValidator,
-  videoChannelsRemoveValidator,
-  videoChannelsNameWithHostValidator,
-  ensureIsLocalChannel,
-  videoChannelsListValidator,
-  videoChannelStatsValidator
-}
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking videoChannelImport parameters', { parameters: req.body })
+
+    if (areValidationErrors(req, res)) return
+
+    if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
+      return res.fail({
+        status: HttpStatusCode.FORBIDDEN_403,
+        message: 'Channel import is impossible as video upload via HTTP is not enabled on the server'
+      })
+    }
+
+    return next()
+  }
+]
 
 // ---------------------------------------------------------------------------