Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

6 files changed, 131 insertions, 3 deletions

diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts
index 8ab750983..1c2315ed1 100644
--- a/shared/server-commands/server/config-command.ts
+++ b/shared/server-commands/server/config-command.ts
@@ -18,17 +18,25 @@ export class ConfigCommand extends AbstractCommand {
     }
   }
 
+  disableImports () {
+    return this.setImportsEnabled(false)
+  }
+
   enableImports () {
+    return this.setImportsEnabled(true)
+  }
+
+  private setImportsEnabled (enabled: boolean) {
     return this.updateExistingSubConfig({
       newConfig: {
         import: {
           videos: {
             http: {
-              enabled: true
+              enabled
             },
 
             torrent: {
-              enabled: true
+              enabled
             }
           }
         }
@@ -36,6 +44,26 @@ export class ConfigCommand extends AbstractCommand {
     })
   }
 
+  private setChannelSyncEnabled (enabled: boolean) {
+    return this.updateExistingSubConfig({
+      newConfig: {
+        import: {
+          videoChannelSynchronization: {
+            enabled
+          }
+        }
+      }
+    })
+  }
+
+  enableChannelSync () {
+    return this.setChannelSyncEnabled(true)
+  }
+
+  disableChannelSync () {
+    return this.setChannelSyncEnabled(false)
+  }
+
   enableLive (options: {
     allowReplay?: boolean
     transcoding?: boolean
@@ -356,6 +384,10 @@ export class ConfigCommand extends AbstractCommand {
           torrent: {
             enabled: false
           }
+        },
+        videoChannelSynchronization: {
+          enabled: false,
+          maxPerUser: 10
         }
       },
       trending: {
diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index 0ad818a11..7acbc978f 100644
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -19,6 +19,7 @@ import {
   CaptionsCommand,
   ChangeOwnershipCommand,
   ChannelsCommand,
+  ChannelSyncsCommand,
   HistoryCommand,
   ImportsCommand,
   LiveCommand,
@@ -118,6 +119,7 @@ export class PeerTubeServer {
   playlists?: PlaylistsCommand
   history?: HistoryCommand
   imports?: ImportsCommand
+  channelSyncs?: ChannelSyncsCommand
   streamingPlaylists?: StreamingPlaylistsCommand
   channels?: ChannelsCommand
   comments?: CommentsCommand
@@ -390,6 +392,7 @@ export class PeerTubeServer {
     this.playlists = new PlaylistsCommand(this)
     this.history = new HistoryCommand(this)
     this.imports = new ImportsCommand(this)
+    this.channelSyncs = new ChannelSyncsCommand(this)
     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
     this.channels = new ChannelsCommand(this)
     this.comments = new CommentsCommand(this)
diff --git a/shared/server-commands/server/servers.ts b/shared/server-commands/server/servers.ts
index 0faee3a8d..29f01774d 100644
--- a/shared/server-commands/server/servers.ts
+++ b/shared/server-commands/server/servers.ts
@@ -39,11 +39,30 @@ async function cleanupTests (servers: PeerTubeServer[]) {
   return Promise.all(p)
 }
 
+function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
+  return {
+    import: {
+      videos: {
+        http: {
+          youtube_dl_release: {
+            url: mode === 'youtube-dl'
+              ? 'https://yt-dl.org/downloads/latest/youtube-dl'
+              : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
+
+            name: mode
+          }
+        }
+      }
+    }
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   createSingleServer,
   createMultipleServers,
   cleanupTests,
-  killallServers
+  killallServers,
+  getServerImportConfig
 }
diff --git a/shared/server-commands/videos/channel-syncs-command.ts b/shared/server-commands/videos/channel-syncs-command.ts
new file mode 100644
index 000000000..de4a160ec
--- /dev/null
+++ b/shared/server-commands/videos/channel-syncs-command.ts
@@ -0,0 +1,55 @@
+import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@shared/models'
+import { pick } from '@shared/core-utils'
+import { unwrapBody } from '../requests'
+import { AbstractCommand, OverrideCommandOptions } from '../shared'
+
+export class ChannelSyncsCommand extends AbstractCommand {
+  private static readonly API_PATH = '/api/v1/video-channel-syncs'
+
+  listByAccount (options: OverrideCommandOptions & {
+    accountName: string
+    start?: number
+    count?: number
+    sort?: string
+  }) {
+    const { accountName, sort = 'createdAt' } = options
+
+    const path = `/api/v1/accounts/${accountName}/video-channel-syncs`
+
+    return this.getRequestBody<ResultList<VideoChannelSync>>({
+      ...options,
+
+      path,
+      query: { sort, ...pick(options, [ 'start', 'count' ]) },
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    })
+  }
+
+  async create (options: OverrideCommandOptions & {
+    attributes: VideoChannelSyncCreate
+  }) {
+    return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({
+      ...options,
+
+      path: ChannelSyncsCommand.API_PATH,
+      fields: options.attributes,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    }))
+  }
+
+  delete (options: OverrideCommandOptions & {
+    channelSyncId: number
+  }) {
+    const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}`
+
+    return this.deleteRequest({
+      ...options,
+
+      path,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+  }
+}
diff --git a/shared/server-commands/videos/channels-command.ts b/shared/server-commands/videos/channels-command.ts
index 8ab124658..a688a120f 100644
--- a/shared/server-commands/videos/channels-command.ts
+++ b/shared/server-commands/videos/channels-command.ts
@@ -181,4 +181,22 @@ export class ChannelsCommand extends AbstractCommand {
       defaultExpectedStatus: HttpStatusCode.OK_200
     })
   }
+
+  importVideos (options: OverrideCommandOptions & {
+    channelName: string
+    externalChannelUrl: string
+  }) {
+    const { channelName, externalChannelUrl } = options
+
+    const path = `/api/v1/video-channels/${channelName}/import-videos`
+
+    return this.postBodyRequest({
+      ...options,
+
+      path,
+      fields: { externalChannelUrl },
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+  }
 }
diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts
index b861731fb..b4d6fa37b 100644
--- a/shared/server-commands/videos/index.ts
+++ b/shared/server-commands/videos/index.ts
@@ -3,6 +3,7 @@ export * from './captions-command'
 export * from './change-ownership-command'
 export * from './channels'
 export * from './channels-command'
+export * from './channel-syncs-command'
 export * from './comments-command'
 export * from './history-command'
 export * from './imports-command'