Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

3 files changed, 88 insertions, 5 deletions

diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts
new file mode 100644
index 000000000..ffdd8c5b5
--- /dev/null
+++ b/server/lib/job-queue/handlers/after-video-channel-import.ts
@@ -0,0 +1,37 @@
+import { Job } from 'bullmq'
+import { logger } from '@server/helpers/logger'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
+
+export async function processAfterVideoChannelImport (job: Job) {
+  const payload = job.data as AfterVideoChannelImportPayload
+  if (!payload.channelSyncId) return
+
+  logger.info('Processing after video channel import in job %s.', job.id)
+
+  const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
+  if (!sync) {
+    logger.error('Unknown sync id %d.', payload.channelSyncId)
+    return
+  }
+
+  const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
+
+  let errors = 0
+  let successes = 0
+
+  for (const value of Object.values(childrenValues)) {
+    if (value.resultType === 'success') successes++
+    else if (value.resultType === 'error') errors++
+  }
+
+  if (errors > 0) {
+    sync.state = VideoChannelSyncState.FAILED
+    logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
+  } else {
+    sync.state = VideoChannelSyncState.SYNCED
+    logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
+  }
+
+  await sync.save()
+}
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
new file mode 100644
index 000000000..9bdb2d269
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -0,0 +1,36 @@
+import { Job } from 'bullmq'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { synchronizeChannel } from '@server/lib/sync-channel'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelImportPayload } from '@shared/models'
+
+export async function processVideoChannelImport (job: Job) {
+  const payload = job.data as VideoChannelImportPayload
+
+  logger.info('Processing video channel import in job %s.', job.id)
+
+  // Channel import requires only http upload to be allowed
+  if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
+    logger.error('Cannot import channel as the HTTP upload is disabled')
+    return
+  }
+
+  if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+    logger.error('Cannot import channel as the synchronization is disabled')
+    return
+  }
+
+  const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
+
+  try {
+    logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
+
+    await synchronizeChannel({
+      channel: videoChannel,
+      externalChannelUrl: payload.externalChannelUrl
+    })
+  } catch (err) {
+    logger.error(`Failed to import channel ${videoChannel.name}`, { err })
+  }
+}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index f4629159c..9901b878c 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import { Hooks } from '@server/lib/plugins/hooks'
 import { ServerConfigManager } from '@server/lib/server-config-manager'
 import { isAbleToUploadVideo } from '@server/lib/user'
-import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video'
+import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
@@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils'
 import {
   ThumbnailType,
   VideoImportPayload,
+  VideoImportPreventExceptionResult,
   VideoImportState,
   VideoImportTorrentPayload,
   VideoImportTorrentPayloadType,
@@ -41,20 +42,29 @@ import { Notifier } from '../../notifier'
 import { generateVideoMiniature } from '../../thumbnail'
 import { JobQueue } from '../job-queue'
 
-async function processVideoImport (job: Job) {
+async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
   const payload = job.data as VideoImportPayload
 
   const videoImport = await getVideoImportOrDie(payload)
   if (videoImport.state === VideoImportState.CANCELLED) {
     logger.info('Do not process import since it has been cancelled', { payload })
-    return
+    return { resultType: 'success' }
   }
 
   videoImport.state = VideoImportState.PROCESSING
   await videoImport.save()
 
-  if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload)
-  if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload)
+  try {
+    if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
+    if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
+
+    return { resultType: 'success' }
+  } catch (err) {
+    if (!payload.preventException) throw err
+
+    logger.warn('Catch error in video import to send value to parent job.', { payload, err })
+    return { resultType: 'error' }
+  }
 }
 
 // ---------------------------------------------------------------------------