Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

4 files changed, 282 insertions, 18 deletions

diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts
new file mode 100644
index 000000000..f7540e1ba
--- /dev/null
+++ b/server/tests/api/videos/channel-import-videos.ts
@@ -0,0 +1,50 @@
+import { expect } from 'chai'
+import { FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled } from '@shared/core-utils'
+import {
+  createSingleServer,
+  getServerImportConfig,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  setDefaultVideoChannel,
+  waitJobs
+} from '@shared/server-commands'
+
+describe('Test videos import in a channel', function () {
+  if (areHttpImportTestsDisabled()) return
+
+  function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
+
+    describe('Import using ' + mode, function () {
+      let server: PeerTubeServer
+
+      before(async function () {
+        this.timeout(120_000)
+
+        server = await createSingleServer(1, getServerImportConfig(mode))
+
+        await setAccessTokensToServers([ server ])
+        await setDefaultVideoChannel([ server ])
+
+        await server.config.enableChannelSync()
+      })
+
+      it('Should import a whole channel', async function () {
+        this.timeout(240_000)
+
+        await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel })
+        await waitJobs(server)
+
+        const videos = await server.videos.listByChannel({ handle: server.store.channel.name })
+        expect(videos.total).to.equal(2)
+      })
+
+      after(async function () {
+        await server?.kill()
+      })
+    })
+  }
+
+  runSuite('yt-dlp')
+  runSuite('youtube-dl')
+})
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index a0b6b01cf..266155297 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -4,6 +4,8 @@ import './single-server'
 import './video-captions'
 import './video-change-ownership'
 import './video-channels'
+import './channel-import-videos'
+import './video-channel-syncs'
 import './video-comments'
 import './video-description'
 import './video-files'
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts
new file mode 100644
index 000000000..229c01f68
--- /dev/null
+++ b/server/tests/api/videos/video-channel-syncs.ts
@@ -0,0 +1,226 @@
+import 'mocha'
+import { expect } from 'chai'
+import { FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled } from '@shared/core-utils'
+import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
+import {
+  ChannelSyncsCommand,
+  createSingleServer,
+  getServerImportConfig,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  setDefaultAccountAvatar,
+  setDefaultChannelAvatar,
+  setDefaultVideoChannel,
+  waitJobs
+} from '@shared/server-commands'
+
+describe('Test channel synchronizations', function () {
+  if (areHttpImportTestsDisabled()) return
+
+  function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
+
+    describe('Sync using ' + mode, function () {
+      let server: PeerTubeServer
+      let command: ChannelSyncsCommand
+      let startTestDate: Date
+      const userInfo = {
+        accessToken: '',
+        username: 'user1',
+        channelName: 'user1_channel',
+        channelId: -1,
+        syncId: -1
+      }
+
+      async function changeDateForSync (channelSyncId: number, newDate: string) {
+        await server.sql.updateQuery(
+          `UPDATE "videoChannelSync" ` +
+          `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
+          `WHERE id=${channelSyncId}`
+        )
+      }
+
+      before(async function () {
+        this.timeout(120_000)
+
+        startTestDate = new Date()
+
+        server = await createSingleServer(1, getServerImportConfig(mode))
+
+        await setAccessTokensToServers([ server ])
+        await setDefaultVideoChannel([ server ])
+        await setDefaultChannelAvatar([ server ])
+        await setDefaultAccountAvatar([ server ])
+
+        await server.config.enableChannelSync()
+
+        command = server.channelSyncs
+
+        {
+          userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
+
+          const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
+          userInfo.channelId = videoChannels[0].id
+        }
+      })
+
+      it('Should fetch the latest channel videos of a remote channel', async function () {
+        this.timeout(120_000)
+
+        {
+          const { video } = await server.imports.importVideo({
+            attributes: {
+              channelId: server.store.channel.id,
+              privacy: VideoPrivacy.PUBLIC,
+              targetUrl: FIXTURE_URLS.youtube
+            }
+          })
+
+          expect(video.name).to.equal('small video - youtube')
+
+          const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+          expect(total).to.equal(1)
+        }
+
+        const { videoChannelSync } = await command.create({
+          attributes: {
+            externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+            videoChannelId: server.store.channel.id
+          },
+          token: server.accessToken,
+          expectedStatus: HttpStatusCode.OK_200
+        })
+
+        // Ensure any missing video not already fetched will be considered as new
+        await changeDateForSync(videoChannelSync.id, '1970-01-01')
+
+        await server.debug.sendCommand({
+          body: {
+            command: 'process-video-channel-sync-latest'
+          }
+        })
+
+        {
+          await waitJobs(server)
+
+          const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+          expect(total).to.equal(2)
+          expect(data[0].name).to.equal('test')
+        }
+      })
+
+      it('Should add another synchronization', async function () {
+        const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
+
+        const { videoChannelSync } = await command.create({
+          attributes: {
+            externalChannelUrl,
+            videoChannelId: server.store.channel.id
+          },
+          token: server.accessToken,
+          expectedStatus: HttpStatusCode.OK_200
+        })
+
+        expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
+        expect(videoChannelSync.channel).to.include({
+          id: server.store.channel.id,
+          name: 'root_channel'
+        })
+        expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
+        expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
+      })
+
+      it('Should add a synchronization for another user', async function () {
+        const { videoChannelSync } = await command.create({
+          attributes: {
+            externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
+            videoChannelId: userInfo.channelId
+          },
+          token: userInfo.accessToken
+        })
+        userInfo.syncId = videoChannelSync.id
+      })
+
+      it('Should not import a channel if not asked', async function () {
+        await waitJobs(server)
+
+        const { data } = await command.listByAccount({ accountName: userInfo.username })
+
+        expect(data[0].state).to.contain({
+          id: VideoChannelSyncState.WAITING_FIRST_RUN,
+          label: 'Waiting first run'
+        })
+      })
+
+      it('Should only fetch the videos newer than the creation date', async function () {
+        this.timeout(120_000)
+
+        await changeDateForSync(userInfo.syncId, '2019-03-01')
+
+        await server.debug.sendCommand({
+          body: {
+            command: 'process-video-channel-sync-latest'
+          }
+        })
+
+        await waitJobs(server)
+
+        const { data, total } = await server.videos.listByChannel({
+          handle: userInfo.channelName,
+          include: VideoInclude.NOT_PUBLISHED_STATE
+        })
+
+        expect(total).to.equal(1)
+        expect(data[0].name).to.equal('test')
+      })
+
+      it('Should list channel synchronizations', async function () {
+        // Root
+        {
+          const { total, data } = await command.listByAccount({ accountName: 'root' })
+          expect(total).to.equal(2)
+
+          expect(data[0]).to.deep.contain({
+            externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+            state: {
+              id: VideoChannelSyncState.SYNCED,
+              label: 'Synchronized'
+            }
+          })
+
+          expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
+
+          expect(data[0].channel).to.contain({ id: server.store.channel.id })
+          expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
+        }
+
+        // User
+        {
+          const { total, data } = await command.listByAccount({ accountName: userInfo.username })
+          expect(total).to.equal(1)
+          expect(data[0]).to.deep.contain({
+            externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
+            state: {
+              id: VideoChannelSyncState.SYNCED,
+              label: 'Synchronized'
+            }
+          })
+        }
+      })
+
+      it('Should remove user\'s channel synchronizations', async function () {
+        await command.delete({ channelSyncId: userInfo.syncId })
+
+        const { total } = await command.listByAccount({ accountName: userInfo.username })
+        expect(total).to.equal(0)
+      })
+
+      after(async function () {
+        await server?.kill()
+      })
+    })
+  }
+
+  runSuite('youtube-dl')
+  runSuite('yt-dlp')
+})
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
index 603e2d234..a487062a2 100644
--- a/server/tests/api/videos/video-imports.ts
+++ b/server/tests/api/videos/video-imports.ts
@@ -12,6 +12,7 @@ import {
   createMultipleServers,
   createSingleServer,
   doubleFollow,
+  getServerImportConfig,
   PeerTubeServer,
   setAccessTokensToServers,
   setDefaultVideoChannel,
@@ -84,24 +85,9 @@ describe('Test video imports', function () {
       let servers: PeerTubeServer[] = []
 
       before(async function () {
-        this.timeout(30_000)
-
-        // Run servers
-        servers = await createMultipleServers(2, {
-          import: {
-            videos: {
-              http: {
-                youtube_dl_release: {
-                  url: mode === 'youtube-dl'
-                    ? 'https://yt-dl.org/downloads/latest/youtube-dl'
-                    : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
-
-                  name: mode
-                }
-              }
-            }
-          }
-        })
+        this.timeout(60_000)
+
+        servers = await createMultipleServers(2, getServerImportConfig(mode))
 
         await setAccessTokensToServers(servers)
         await setDefaultVideoChannel(servers)