Fix sync import of latest videos

5 files changed, 106 insertions, 47 deletions

diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
index 303e4051f..a74904e43 100644
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
@@ -12,7 +12,7 @@ type YoutubeDLInfo = {
   tags?: string[]
   thumbnailUrl?: string
   ext?: string
-  originallyPublishedAt?: Date
+  originallyPublishedAtWithoutTime?: Date
   webpageUrl?: string
 
   urls?: string[]
@@ -81,7 +81,7 @@ class YoutubeDLInfoBuilder {
       tags: this.getTags(obj.tags),
       thumbnailUrl: obj.thumbnail || undefined,
       urls: this.buildAvailableUrl(obj),
-      originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
+      originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
       ext: obj.ext,
       webpageUrl: obj.webpage_url
     }
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
index 9aaad659e..600292844 100644
--- a/server/lib/job-queue/handlers/video-channel-import.ts
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
 import { VideoChannelModel } from '@server/models/video/video-channel'
 import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 import { MChannelSync } from '@server/types/models'
-import { VideoChannelImportPayload } from '@shared/models'
+import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models'
 
 export async function processVideoChannelImport (job: Job) {
   const payload = job.data as VideoChannelImportPayload
@@ -42,5 +42,7 @@ export async function processVideoChannelImport (job: Job) {
     })
   } catch (err) {
     logger.error(`Failed to import channel ${videoChannel.name}`, { err })
+    channelSync.state = VideoChannelSyncState.FAILED
+    await channelSync.save()
   }
 }
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index 3a81daac0..6cd1fc26d 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -93,7 +93,10 @@ async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Dat
 
     const videoInfo = await youtubeDL.getInfoForDownload()
 
-    if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
+    const onlyAfterWithoutTime = new Date(onlyAfter)
+    onlyAfterWithoutTime.setHours(0, 0, 0)
+
+    if (videoInfo.originallyPublishedAtWithoutTime.getTime() >= onlyAfterWithoutTime.getTime()) {
       return true
     }
   }
diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts
index de95116aa..d9da09040 100644
--- a/server/lib/video-import.ts
+++ b/server/lib/video-import.ts
@@ -125,7 +125,7 @@ async function buildVideoFromImport ({ channelId, importData, importDataOverride
     channelId,
     originallyPublishedAt: importDataOverride?.originallyPublishedAt
       ? new Date(importDataOverride?.originallyPublishedAt)
-      : importData.originallyPublishedAt
+      : importData.originallyPublishedAtWithoutTime
   }
 
   videoData = await Hooks.wrapObject(
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts
index 4e18ddfc7..65b143a4f 100644
--- a/server/tests/api/videos/video-channel-syncs.ts
+++ b/server/tests/api/videos/video-channel-syncs.ts
@@ -3,9 +3,9 @@ import { FIXTURE_URLS } from '@server/tests/shared'
 import { areHttpImportTestsDisabled } from '@shared/core-utils'
 import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
 import {
-  ChannelSyncsCommand,
-  createSingleServer,
+  createMultipleServers,
   getServerImportConfig,
+  killallServers,
   PeerTubeServer,
   setAccessTokensToServers,
   setDefaultAccountAvatar,
@@ -20,8 +20,7 @@ describe('Test channel synchronizations', function () {
   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
 
     describe('Sync using ' + mode, function () {
-      let server: PeerTubeServer
-      let command: ChannelSyncsCommand
+      let servers: PeerTubeServer[]
 
       let startTestDate: Date
 
@@ -35,7 +34,7 @@ describe('Test channel synchronizations', function () {
       }
 
       async function changeDateForSync (channelSyncId: number, newDate: string) {
-        await server.sql.updateQuery(
+        await servers[0].sql.updateQuery(
           `UPDATE "videoChannelSync" ` +
           `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
           `WHERE id=${channelSyncId}`
@@ -43,25 +42,23 @@ describe('Test channel synchronizations', function () {
       }
 
       before(async function () {
-        this.timeout(120_000)
+        this.timeout(240_000)
 
         startTestDate = new Date()
 
-        server = await createSingleServer(1, getServerImportConfig(mode))
-
-        await setAccessTokensToServers([ server ])
-        await setDefaultVideoChannel([ server ])
-        await setDefaultChannelAvatar([ server ])
-        await setDefaultAccountAvatar([ server ])
+        servers = await createMultipleServers(2, getServerImportConfig(mode))
 
-        await server.config.enableChannelSync()
+        await setAccessTokensToServers(servers)
+        await setDefaultVideoChannel(servers)
+        await setDefaultChannelAvatar(servers)
+        await setDefaultAccountAvatar(servers)
 
-        command = server.channelSyncs
+        await servers[0].config.enableChannelSync()
 
         {
-          userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
+          userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username)
 
-          const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
+          const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken })
           userInfo.channelId = videoChannels[0].id
         }
       })
@@ -70,9 +67,9 @@ describe('Test channel synchronizations', function () {
         this.timeout(120_000)
 
         {
-          const { video } = await server.imports.importVideo({
+          const { video } = await servers[0].imports.importVideo({
             attributes: {
-              channelId: server.store.channel.id,
+              channelId: servers[0].store.channel.id,
               privacy: VideoPrivacy.PUBLIC,
               targetUrl: FIXTURE_URLS.youtube
             }
@@ -80,16 +77,16 @@ describe('Test channel synchronizations', function () {
 
           expect(video.name).to.equal('small video - youtube')
 
-          const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+          const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
           expect(total).to.equal(1)
         }
 
-        const { videoChannelSync } = await command.create({
+        const { videoChannelSync } = await servers[0].channelSyncs.create({
           attributes: {
             externalChannelUrl: FIXTURE_URLS.youtubeChannel,
-            videoChannelId: server.store.channel.id
+            videoChannelId: servers[0].store.channel.id
           },
-          token: server.accessToken,
+          token: servers[0].accessToken,
           expectedStatus: HttpStatusCode.OK_200
         })
         rootChannelSyncId = videoChannelSync.id
@@ -97,16 +94,19 @@ describe('Test channel synchronizations', function () {
         // Ensure any missing video not already fetched will be considered as new
         await changeDateForSync(videoChannelSync.id, '1970-01-01')
 
-        await server.debug.sendCommand({
+        await servers[0].debug.sendCommand({
           body: {
             command: 'process-video-channel-sync-latest'
           }
         })
 
         {
-          await waitJobs(server)
+          await waitJobs(servers)
 
-          const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+          const { total, data } = await servers[0].videos.listByChannel({
+            handle: 'root_channel',
+            include: VideoInclude.NOT_PUBLISHED_STATE
+          })
           expect(total).to.equal(2)
           expect(data[0].name).to.equal('test')
         }
@@ -115,18 +115,18 @@ describe('Test channel synchronizations', function () {
       it('Should add another synchronization', async function () {
         const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
 
-        const { videoChannelSync } = await command.create({
+        const { videoChannelSync } = await servers[0].channelSyncs.create({
           attributes: {
             externalChannelUrl,
-            videoChannelId: server.store.channel.id
+            videoChannelId: servers[0].store.channel.id
           },
-          token: server.accessToken,
+          token: servers[0].accessToken,
           expectedStatus: HttpStatusCode.OK_200
         })
 
         expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
         expect(videoChannelSync.channel).to.include({
-          id: server.store.channel.id,
+          id: servers[0].store.channel.id,
           name: 'root_channel'
         })
         expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
@@ -134,7 +134,7 @@ describe('Test channel synchronizations', function () {
       })
 
       it('Should add a synchronization for another user', async function () {
-        const { videoChannelSync } = await command.create({
+        const { videoChannelSync } = await servers[0].channelSyncs.create({
           attributes: {
             externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
             videoChannelId: userInfo.channelId
@@ -145,9 +145,9 @@ describe('Test channel synchronizations', function () {
       })
 
       it('Should not import a channel if not asked', async function () {
-        await waitJobs(server)
+        await waitJobs(servers)
 
-        const { data } = await command.listByAccount({ accountName: userInfo.username })
+        const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
 
         expect(data[0].state).to.contain({
           id: VideoChannelSyncState.WAITING_FIRST_RUN,
@@ -160,15 +160,15 @@ describe('Test channel synchronizations', function () {
 
         await changeDateForSync(userInfo.syncId, '2019-03-01')
 
-        await server.debug.sendCommand({
+        await servers[0].debug.sendCommand({
           body: {
             command: 'process-video-channel-sync-latest'
           }
         })
 
-        await waitJobs(server)
+        await waitJobs(servers)
 
-        const { data, total } = await server.videos.listByChannel({
+        const { data, total } = await servers[0].videos.listByChannel({
           handle: userInfo.channelName,
           include: VideoInclude.NOT_PUBLISHED_STATE
         })
@@ -180,7 +180,7 @@ describe('Test channel synchronizations', function () {
       it('Should list channel synchronizations', async function () {
         // Root
         {
-          const { total, data } = await command.listByAccount({ accountName: 'root' })
+          const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
           expect(total).to.equal(2)
 
           expect(data[0]).to.deep.contain({
@@ -193,13 +193,13 @@ describe('Test channel synchronizations', function () {
 
           expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
 
-          expect(data[0].channel).to.contain({ id: server.store.channel.id })
+          expect(data[0].channel).to.contain({ id: servers[0].store.channel.id })
           expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
         }
 
         // User
         {
-          const { total, data } = await command.listByAccount({ accountName: userInfo.username })
+          const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
           expect(total).to.equal(1)
           expect(data[0]).to.deep.contain({
             externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
@@ -212,7 +212,7 @@ describe('Test channel synchronizations', function () {
       })
 
       it('Should list imports of a channel synchronization', async function () {
-        const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
+        const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
 
         expect(total).to.equal(1)
         expect(data).to.have.lengthOf(1)
@@ -220,14 +220,68 @@ describe('Test channel synchronizations', function () {
       })
 
       it('Should remove user\'s channel synchronizations', async function () {
-        await command.delete({ channelSyncId: userInfo.syncId })
+        await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId })
 
-        const { total } = await command.listByAccount({ accountName: userInfo.username })
+        const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
         expect(total).to.equal(0)
       })
 
+      // FIXME: youtube-dl doesn't work when speicifying a port after the hostname
+      // it('Should import a remote PeerTube channel', async function () {
+      //   this.timeout(240_000)
+
+      //   await servers[1].videos.quickUpload({ name: 'remote 1' })
+      //   await waitJobs(servers)
+
+      //   const { videoChannelSync } = await servers[0].channelSyncs.create({
+      //     attributes: {
+      //       externalChannelUrl: servers[1].url + '/c/root_channel',
+      //       videoChannelId: userInfo.channelId
+      //     },
+      //     token: userInfo.accessToken
+      //   })
+      //   await servers[0].channels.importVideos({
+      //     channelName: userInfo.channelName,
+      //     externalChannelUrl: servers[1].url + '/c/root_channel',
+      //     videoChannelSyncId: videoChannelSync.id,
+      //     token: userInfo.accessToken
+      //   })
+
+      //   await waitJobs(servers)
+
+      //   const { data, total } = await servers[0].videos.listByChannel({
+      //     handle: userInfo.channelName,
+      //     include: VideoInclude.NOT_PUBLISHED_STATE
+      //   })
+
+      //   expect(total).to.equal(2)
+      //   expect(data[0].name).to.equal('remote 1')
+      // })
+
+      // it('Should keep synced a remote PeerTube channel', async function () {
+      //   this.timeout(240_000)
+
+      //   await servers[1].videos.quickUpload({ name: 'remote 2' })
+      //   await waitJobs(servers)
+
+      //   await servers[0].debug.sendCommand({
+      //     body: {
+      //       command: 'process-video-channel-sync-latest'
+      //     }
+      //   })
+
+      //   await waitJobs(servers)
+
+      //   const { data, total } = await servers[0].videos.listByChannel({
+      //     handle: userInfo.channelName,
+      //     include: VideoInclude.NOT_PUBLISHED_STATE
+      //   })
+      //   expect(total).to.equal(2)
+      //   expect(data[0].name).to.equal('remote 2')
+      // })
+
       after(async function () {
-        await server?.kill()
+        await killallServers(servers)
       })
     })
   }