Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

6 files changed, 82 insertions, 5 deletions

diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index f0f16d9bd..74c82541e 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -48,6 +48,7 @@ function checkConfig () {
   checkRemoteRedundancyConfig()
   checkStorageConfig()
   checkTranscodingConfig()
+  checkImportConfig()
   checkBroadcastMessageConfig()
   checkSearchConfig()
   checkLiveConfig()
@@ -200,6 +201,12 @@ function checkTranscodingConfig () {
   }
 }
 
+function checkImportConfig () {
+  if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) {
+    throw new Error('You need to enable HTTP import to allow synchronization')
+  }
+}
+
 function checkBroadcastMessageConfig () {
   if (CONFIG.BROADCAST_MESSAGE.ENABLED) {
     const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index f4057b81b..3188903be 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -32,6 +32,8 @@ function checkMissedConfig () {
     'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
     'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
     'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
+    'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
+    'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
     'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
     'client.videos.miniature.display_author_avatar',
     'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 1a0b8942c..2c92bea22 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -398,6 +398,14 @@ const CONFIG = {
       TORRENT: {
         get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
       }
+    },
+    VIDEO_CHANNEL_SYNCHRONIZATION: {
+      get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') },
+      get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') },
+      get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
+      get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
+        return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
+      }
     }
   },
   AUTO_BLACKLIST: {
@@ -499,6 +507,7 @@ const CONFIG = {
       get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
     }
   }
+
 }
 
 function registerConfigChangedHandler (fun: Function) {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5a5f2d666..697a64d42 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
 import {
   AbuseState,
   JobType,
+  VideoChannelSyncState,
   VideoImportState,
   VideoPrivacy,
   VideoRateType,
@@ -24,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 725
+const LAST_MIGRATION_VERSION = 730
 
 // ---------------------------------------------------------------------------
 
@@ -64,6 +65,7 @@ const SORTABLE_COLUMNS = {
   JOBS: [ 'createdAt' ],
   VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
   VIDEO_IMPORTS: [ 'createdAt' ],
+  VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ],
 
   VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
   VIDEO_COMMENTS: [ 'createdAt' ],
@@ -156,6 +158,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'video-live-ending': 1,
   'video-studio-edition': 1,
   'manage-video-torrent': 1,
+  'video-channel-import': 1,
+  'after-video-channel-import': 1,
   'move-to-object-storage': 3,
   'notify': 1,
   'federate-video': 1
@@ -178,6 +182,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
   'video-studio-edition': 1,
   'manage-video-torrent': 1,
   'move-to-object-storage': 1,
+  'video-channel-import': 1,
+  'after-video-channel-import': 1,
   'notify': 5,
   'federate-video': 3
 }
@@ -199,9 +205,11 @@ const JOB_TTL: { [id in JobType]: number } = {
   'video-redundancy': 1000 * 3600 * 3, // 3 hours
   'video-live-ending': 1000 * 60 * 10, // 10 minutes
   'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
+  'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
+  'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
+  'after-video-channel-import': 60000 * 5, // 5 minutes
   'notify': 60000 * 5, // 5 minutes
-  'federate-video': 60000 * 5, // 5 minutes
-  'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
+  'federate-video': 60000 * 5 // 5 minutes
 }
 const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
   'videos-views-stats': {
@@ -246,7 +254,8 @@ const SCHEDULER_INTERVALS_MS = {
   REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
   REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
   UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
-  REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour
+  REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
+  CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
 }
 
 // ---------------------------------------------------------------------------
@@ -276,8 +285,12 @@ const CONSTRAINTS_FIELDS = {
     NAME: { min: 1, max: 120 }, // Length
     DESCRIPTION: { min: 3, max: 1000 }, // Length
     SUPPORT: { min: 3, max: 1000 }, // Length
+    EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length
     URL: { min: 3, max: 2000 } // Length
   },
+  VIDEO_CHANNEL_SYNCS: {
+    EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length
+  },
   VIDEO_CAPTIONS: {
     CAPTION_FILE: {
       EXTNAME: [ '.vtt', '.srt' ],
@@ -478,6 +491,13 @@ const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
   [VideoImportState.PROCESSING]: 'Processing'
 }
 
+const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = {
+  [VideoChannelSyncState.FAILED]: 'Failed',
+  [VideoChannelSyncState.SYNCED]: 'Synchronized',
+  [VideoChannelSyncState.PROCESSING]: 'Processing',
+  [VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run'
+}
+
 const ABUSE_STATES: { [ id in AbuseState ]: string } = {
   [AbuseState.PENDING]: 'Pending',
   [AbuseState.REJECTED]: 'Rejected',
@@ -1005,6 +1025,7 @@ export {
   JOB_COMPLETED_LIFETIME,
   HTTP_SIGNATURE,
   VIDEO_IMPORT_STATES,
+  VIDEO_CHANNEL_SYNC_STATE,
   VIEW_LIFETIME,
   CONTACT_FORM_LIFETIME,
   VIDEO_PLAYLIST_PRIVACIES,
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 91286241b..f55f40df0 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -50,6 +50,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
 import { VideoTagModel } from '../models/video/video-tag'
 import { VideoViewModel } from '../models/view/video-view'
 import { CONFIG } from './config'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -153,7 +154,8 @@ async function initDatabaseModels (silent: boolean) {
     VideoTrackerModel,
     PluginModel,
     ActorCustomPageModel,
-    VideoJobInfoModel
+    VideoJobInfoModel,
+    VideoChannelSyncModel
   ])
 
   // Check extensions exist in the database
diff --git a/server/initializers/migrations/0730-video-channel-sync.ts b/server/initializers/migrations/0730-video-channel-sync.ts
new file mode 100644
index 000000000..a2fe8211f
--- /dev/null
+++ b/server/initializers/migrations/0730-video-channel-sync.ts
@@ -0,0 +1,36 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+  db: any
+}): Promise<void> {
+  const query = `
+    CREATE TABLE IF NOT EXISTS "videoChannelSync" (
+      "id"   SERIAL,
+      "externalChannelUrl" VARCHAR(2000) NOT NULL DEFAULT NULL,
+      "videoChannelId" INTEGER NOT NULL REFERENCES "videoChannel" ("id")
+        ON DELETE CASCADE
+        ON UPDATE CASCADE,
+      "state" INTEGER NOT NULL DEFAULT 1,
+      "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+      "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+      "lastSyncAt" TIMESTAMP WITH TIME ZONE,
+      PRIMARY KEY ("id")
+    );
+  `
+  await utils.sequelize.query(query, { transaction: utils.transaction })
+}
+
+async function down (utils: {
+  queryInterface: Sequelize.QueryInterface
+  transaction: Sequelize.Transaction
+}) {
+  await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction })
+}
+
+export {
+  up,
+  down
+}