Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

6 files changed, 478 insertions, 14 deletions

diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 2f9f553ab..d67e51123 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -1,7 +1,8 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import 'mocha'
-import { omit } from 'lodash'
+import { merge, omit } from 'lodash'
+import { CustomConfig, HttpStatusCode } from '@shared/models'
 import {
   cleanupTests,
   createSingleServer,
@@ -11,7 +12,6 @@ import {
   PeerTubeServer,
   setAccessTokensToServers
 } from '@shared/server-commands'
-import { CustomConfig, HttpStatusCode } from '@shared/models'
 
 describe('Test config API validators', function () {
   const path = '/api/v1/config/custom'
@@ -162,6 +162,10 @@ describe('Test config API validators', function () {
         torrent: {
           enabled: false
         }
+      },
+      videoChannelSynchronization: {
+        enabled: false,
+        maxPerUser: 10
       }
     },
     trending: {
@@ -346,7 +350,26 @@ describe('Test config API validators', function () {
       })
     })
 
-    it('Should success with the correct parameters', async function () {
+    it('Should fail with a disabled http upload & enabled sync', async function () {
+      const newUpdateParams: CustomConfig = merge({}, updateParams, {
+        import: {
+          videos: {
+            http: { enabled: false }
+          },
+          videoChannelSynchronization: { enabled: true }
+        }
+      })
+
+      await makePutBodyRequest({
+        url: server.url,
+        path,
+        fields: newUpdateParams,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should succeed with the correct parameters', async function () {
       await makePutBodyRequest({
         url: server.url,
         path,
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index a27bc8509..5f1168b53 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -27,6 +27,7 @@ import './video-channels'
 import './video-comments'
 import './video-files'
 import './video-imports'
+import './video-channel-syncs'
 import './video-playlists'
 import './video-source'
 import './video-studio'
diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts
index deb4a7aa3..f64eafc18 100644
--- a/server/tests/api/check-params/upload-quota.ts
+++ b/server/tests/api/check-params/upload-quota.ts
@@ -70,7 +70,7 @@ describe('Test upload quota', function () {
     })
 
     it('Should fail to import with HTTP/Torrent/magnet', async function () {
-      this.timeout(120000)
+      this.timeout(120_000)
 
       const baseAttributes = {
         channelId: server.store.channel.id,
diff --git a/server/tests/api/check-params/video-channel-syncs.ts b/server/tests/api/check-params/video-channel-syncs.ts
new file mode 100644
index 000000000..bcd8984df
--- /dev/null
+++ b/server/tests/api/check-params/video-channel-syncs.ts
@@ -0,0 +1,318 @@
+import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
+import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
+import {
+  ChannelSyncsCommand,
+  createSingleServer,
+  makePostBodyRequest,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  setDefaultVideoChannel
+} from '@shared/server-commands'
+
+describe('Test video channel sync API validator', () => {
+  const path = '/api/v1/video-channel-syncs'
+  let server: PeerTubeServer
+  let command: ChannelSyncsCommand
+  let rootChannelId: number
+  let rootChannelSyncId: number
+  const userInfo = {
+    accessToken: '',
+    username: 'user1',
+    id: -1,
+    channelId: -1,
+    syncId: -1
+  }
+
+  async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> {
+    try {
+      await server.config.disableChannelSync()
+      await callback()
+    } finally {
+      await server.config.enableChannelSync()
+    }
+  }
+
+  async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> {
+    const origConfig = await server.config.getCustomConfig()
+
+    await server.config.updateExistingSubConfig({
+      newConfig: {
+        import: {
+          videoChannelSynchronization: {
+            maxPerUser: maxSync
+          }
+        }
+      }
+    })
+
+    try {
+      await callback()
+    } finally {
+      await server.config.updateCustomConfig({ newCustomConfig: origConfig })
+    }
+  }
+
+  before(async function () {
+    this.timeout(30_000)
+
+    server = await createSingleServer(1)
+
+    await setAccessTokensToServers([ server ])
+    await setDefaultVideoChannel([ server ])
+
+    command = server.channelSyncs
+
+    rootChannelId = server.store.channel.id
+
+    {
+      userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
+
+      const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken })
+      userInfo.id = userId
+      userInfo.channelId = videoChannels[0].id
+    }
+
+    await server.config.enableChannelSync()
+  })
+
+  describe('When creating a sync', function () {
+    let baseCorrectParams: VideoChannelSyncCreate
+
+    before(function () {
+      baseCorrectParams = {
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        videoChannelId: rootChannelId
+      }
+    })
+
+    it('Should fail when sync is disabled', async function () {
+      await withChannelSyncDisabled(async () => {
+        await command.create({
+          token: server.accessToken,
+          attributes: baseCorrectParams,
+          expectedStatus: HttpStatusCode.FORBIDDEN_403
+        })
+      })
+    })
+
+    it('Should fail with nothing', async function () {
+      const fields = {}
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        token: server.accessToken,
+        fields,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail with no authentication', async function () {
+      await command.create({
+        token: null,
+        attributes: baseCorrectParams,
+        expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+      })
+    })
+
+    it('Should fail without a target url', async function () {
+      const attributes: VideoChannelSyncCreate = {
+        ...baseCorrectParams,
+        externalChannelUrl: null
+      }
+      await command.create({
+        token: server.accessToken,
+        attributes,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail without a channelId', async function () {
+      const attributes: VideoChannelSyncCreate = {
+        ...baseCorrectParams,
+        videoChannelId: null
+      }
+      await command.create({
+        token: server.accessToken,
+        attributes,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail with a channelId refering nothing', async function () {
+      const attributes: VideoChannelSyncCreate = {
+        ...baseCorrectParams,
+        videoChannelId: 42
+      }
+      await command.create({
+        token: server.accessToken,
+        attributes,
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should fail to create a sync when the user does not own the channel', async function () {
+      await command.create({
+        token: userInfo.accessToken,
+        attributes: baseCorrectParams,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+    })
+
+    it('Should succeed to create a sync with root and for another user\'s channel', async function () {
+      const { videoChannelSync } = await command.create({
+        token: server.accessToken,
+        attributes: {
+          ...baseCorrectParams,
+          videoChannelId: userInfo.channelId
+        },
+        expectedStatus: HttpStatusCode.OK_200
+      })
+      userInfo.syncId = videoChannelSync.id
+    })
+
+    it('Should succeed with the correct parameters', async function () {
+      const { videoChannelSync } = await command.create({
+        token: server.accessToken,
+        attributes: baseCorrectParams,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+      rootChannelSyncId = videoChannelSync.id
+    })
+
+    it('Should fail when the user exceeds allowed number of synchronizations', async function () {
+      await withMaxSyncsPerUser(1, async () => {
+        await command.create({
+          token: server.accessToken,
+          attributes: {
+            ...baseCorrectParams,
+            videoChannelId: userInfo.channelId
+          },
+          expectedStatus: HttpStatusCode.BAD_REQUEST_400
+        })
+      })
+    })
+  })
+
+  describe('When listing my channel syncs', function () {
+    const myPath = '/api/v1/accounts/root/video-channel-syncs'
+
+    it('Should fail with a bad start pagination', async function () {
+      await checkBadStartPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should fail with a bad count pagination', async function () {
+      await checkBadCountPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should fail with an incorrect sort', async function () {
+      await checkBadSortPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should succeed with the correct parameters', async function () {
+      await command.listByAccount({
+        accountName: 'root',
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+    })
+
+    it('Should fail with no authentication', async function () {
+      await command.listByAccount({
+        accountName: 'root',
+        token: null,
+        expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+      })
+    })
+
+    it('Should fail when a simple user lists another user\'s synchronizations', async function () {
+      await command.listByAccount({
+        accountName: 'root',
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+    })
+
+    it('Should succeed when root lists another user\'s synchronizations', async function () {
+      await command.listByAccount({
+        accountName: userInfo.username,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+    })
+
+    it('Should succeed even with synchronization disabled', async function () {
+      await withChannelSyncDisabled(async function () {
+        await command.listByAccount({
+          accountName: 'root',
+          token: server.accessToken,
+          expectedStatus: HttpStatusCode.OK_200
+        })
+      })
+    })
+  })
+
+  describe('When triggering deletion', function () {
+    it('should fail with no authentication', async function () {
+      await command.delete({
+        channelSyncId: userInfo.syncId,
+        token: null,
+        expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+      })
+    })
+
+    it('Should fail when channelSyncId does not refer to any sync', async function () {
+      await command.delete({
+        channelSyncId: 42,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should fail when sync is not owned by the user', async function () {
+      await command.delete({
+        channelSyncId: rootChannelSyncId,
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+    })
+
+    it('Should succeed when root delete a sync they do not own', async function () {
+      await command.delete({
+        channelSyncId: userInfo.syncId,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.NO_CONTENT_204
+      })
+    })
+
+    it('should succeed when user delete a sync they own', async function () {
+      const { videoChannelSync } = await command.create({
+        attributes: {
+          externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+          videoChannelId: userInfo.channelId
+        },
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+
+      await command.delete({
+        channelSyncId: videoChannelSync.id,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.NO_CONTENT_204
+      })
+    })
+
+    it('Should succeed even when synchronization is disabled', async function () {
+      await withChannelSyncDisabled(async function () {
+        await command.delete({
+          channelSyncId: rootChannelSyncId,
+          token: server.accessToken,
+          expectedStatus: HttpStatusCode.NO_CONTENT_204
+        })
+      })
+    })
+  })
+
+  after(async function () {
+    await server?.kill()
+  })
+})
diff --git a/server/tests/api/check-params/video-channels.ts b/server/tests/api/check-params/video-channels.ts
index 5c2650fac..337ea1dd4 100644
--- a/server/tests/api/check-params/video-channels.ts
+++ b/server/tests/api/check-params/video-channels.ts
@@ -3,8 +3,8 @@
 import 'mocha'
 import * as chai from 'chai'
 import { omit } from 'lodash'
-import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared'
-import { buildAbsoluteFixturePath } from '@shared/core-utils'
+import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils'
 import { HttpStatusCode, VideoChannelUpdate } from '@shared/models'
 import {
   ChannelsCommand,
@@ -23,7 +23,13 @@ const expect = chai.expect
 describe('Test video channels API validator', function () {
   const videoChannelPath = '/api/v1/video-channels'
   let server: PeerTubeServer
-  let accessTokenUser: string
+  const userInfo = {
+    accessToken: '',
+    channelName: 'fake_channel',
+    id: -1,
+    videoQuota: -1,
+    videoQuotaDaily: -1
+  }
   let command: ChannelsCommand
 
   // ---------------------------------------------------------------
@@ -35,14 +41,15 @@ describe('Test video channels API validator', function () {
 
     await setAccessTokensToServers([ server ])
 
-    const user = {
+    const userCreds = {
       username: 'fake',
       password: 'fake_password'
     }
 
     {
-      await server.users.create({ username: user.username, password: user.password })
-      accessTokenUser = await server.login.getAccessToken(user)
+      const user = await server.users.create({ username: userCreds.username, password: userCreds.password })
+      userInfo.id = user.id
+      userInfo.accessToken = await server.login.getAccessToken(userCreds)
     }
 
     command = server.channels
@@ -191,7 +198,7 @@ describe('Test video channels API validator', function () {
       await makePutBodyRequest({
         url: server.url,
         path,
-        token: accessTokenUser,
+        token: userInfo.accessToken,
         fields: baseCorrectParams,
         expectedStatus: HttpStatusCode.FORBIDDEN_403
       })
@@ -339,7 +346,7 @@ describe('Test video channels API validator', function () {
     })
 
     it('Should fail with a another user', async function () {
-      await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
+      await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
     })
 
     it('Should succeed with the correct params', async function () {
@@ -347,13 +354,122 @@ describe('Test video channels API validator', function () {
     })
   })
 
+  describe('When triggering full synchronization', function () {
+
+    it('Should fail when HTTP upload is disabled', async function () {
+      await server.config.disableImports()
+
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+
+      await server.config.enableImports()
+    })
+
+    it('Should fail when externalChannelUrl is not provided', async function () {
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: null,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail when externalChannelUrl is malformed', async function () {
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: 'not-a-url',
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail with no authentication', async function () {
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: null,
+        expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+      })
+    })
+
+    it('Should fail when sync is not owned by the user', async function () {
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+    })
+
+    it('Should fail when the user has no quota', async function () {
+      await server.users.update({
+        userId: userInfo.id,
+        videoQuota: 0
+      })
+
+      await command.importVideos({
+        channelName: 'fake_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
+      })
+
+      await server.users.update({
+        userId: userInfo.id,
+        videoQuota: userInfo.videoQuota
+      })
+    })
+
+    it('Should fail when the user has no daily quota', async function () {
+      await server.users.update({
+        userId: userInfo.id,
+        videoQuotaDaily: 0
+      })
+
+      await command.importVideos({
+        channelName: 'fake_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
+      })
+
+      await server.users.update({
+        userId: userInfo.id,
+        videoQuotaDaily: userInfo.videoQuotaDaily
+      })
+    })
+
+    it('Should succeed when sync is run by its owner', async function () {
+      if (!areHttpImportTestsDisabled()) return
+
+      await command.importVideos({
+        channelName: 'fake_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: userInfo.accessToken
+      })
+    })
+
+    it('Should succeed when sync is run with root and for another user\'s channel', async function () {
+      if (!areHttpImportTestsDisabled()) return
+
+      await command.importVideos({
+        channelName: 'fake_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel
+      })
+    })
+  })
+
   describe('When deleting a video channel', function () {
     it('Should fail with a non authenticated user', async function () {
       await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
     })
 
     it('Should fail with another authenticated user', async function () {
-      await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
+      await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
     })
 
     it('Should fail with an unknown video channel id', async function () {
diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts
index 4439810e8..5cdd0d925 100644
--- a/server/tests/api/check-params/video-imports.ts
+++ b/server/tests/api/check-params/video-imports.ts
@@ -88,7 +88,13 @@ describe('Test video imports API validator', function () {
 
     it('Should fail with nothing', async function () {
       const fields = {}
-      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        token: server.accessToken,
+        fields,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
     })
 
     it('Should fail without a target url', async function () {