From 4efa5535ccc947597b7a8e343d6665229ab9cfce Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Nov 2022 11:11:39 +0100 Subject: Fix infinite playlist import Using an hard videos limit in config --- config/default.yaml | 3 ++ config/production.yaml.example | 3 ++ server/initializers/checker-before-init.ts | 1 + server/initializers/config.ts | 3 ++ .../lib/job-queue/handlers/video-channel-import.ts | 3 +- server/lib/sync-channel.ts | 2 +- server/tests/api/videos/channel-import-videos.ts | 43 +++++++++++++++++++++- server/tests/api/videos/video-channel-syncs.ts | 2 +- 8 files changed, 56 insertions(+), 4 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 2e249cc76..a916b1dc3 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -578,6 +578,9 @@ import: # Number of latest published videos to check and to potentially import when syncing a channel videos_limit_per_synchronization: 10 + # Max number of videos to import when the user asks for full sync + full_sync_videos_limit: 1000 + auto_blacklist: # New videos automatically blacklisted so moderators can review before publishing videos: diff --git a/config/production.yaml.example b/config/production.yaml.example index c136a73ad..100bc7948 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -588,6 +588,9 @@ import: # Number of latest published videos to check and to potentially import when syncing a channel videos_limit_per_synchronization: 10 + # Max number of videos to import when the user asks for full sync + full_sync_videos_limit: 1000 + auto_blacklist: # New videos automatically blacklisted so moderators can review before publishing videos: diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index e5ec0d1df..42be7ee6e 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -35,6 +35,7 @@ function checkMissedConfig () { 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', + 'import.video_channel_synchronization.full_sync_videos_limit', 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', 'client.videos.miniature.display_author_avatar', 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 7652da24a..3dd1f6971 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -418,6 +418,9 @@ const CONFIG = { get CHECK_INTERVAL () { return parseDurationToMs(config.get('import.video_channel_synchronization.check_interval')) }, get VIDEOS_LIMIT_PER_SYNCHRONIZATION () { return config.get('import.video_channel_synchronization.videos_limit_per_synchronization') + }, + get FULL_SYNC_VIDEOS_LIMIT () { + return config.get('import.video_channel_synchronization.full_sync_videos_limit') } } }, diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts index c3dd8a688..035f88e96 100644 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ b/server/lib/job-queue/handlers/video-channel-import.ts @@ -37,6 +37,7 @@ export async function processVideoChannelImport (job: Job) { await synchronizeChannel({ channel: videoChannel, externalChannelUrl: payload.externalChannelUrl, - channelSync + channelSync, + videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT }) } diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts index 4d00d6163..10167ee38 100644 --- a/server/lib/sync-channel.ts +++ b/server/lib/sync-channel.ts @@ -12,8 +12,8 @@ import { ServerConfigManager } from './server-config-manager' export async function synchronizeChannel (options: { channel: MChannelAccountDefault externalChannelUrl: string + videosCountLimit: number channelSync?: MChannelSync - videosCountLimit?: number onlyAfter?: Date }) { const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts index 7cfd02fbb..a66f88a0e 100644 --- a/server/tests/api/videos/channel-import-videos.ts +++ b/server/tests/api/videos/channel-import-videos.ts @@ -109,6 +109,45 @@ describe('Test videos import in a channel', function () { } }) + it('Should limit max amount of videos synced on full sync', async function () { + this.timeout(240_000) + + await server.kill() + await server.run({ + import: { + video_channel_synchronization: { + full_sync_videos_limit: 1 + } + } + }) + + const { id } = await server.channels.create({ attributes: { name: 'channel3' } }) + const channel3Id = id + + const { videoChannelSync } = await server.channelSyncs.create({ + attributes: { + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelId: channel3Id + } + }) + const syncId = videoChannelSync.id + + await waitJobs(server) + + await server.channels.importVideos({ + channelName: 'channel3', + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelSyncId: syncId + }) + + await waitJobs(server) + + const { total, data } = await server.videos.listByChannel({ handle: 'channel3' }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + after(async function () { await server?.kill() }) @@ -116,5 +155,7 @@ describe('Test videos import in a channel', function () { } runSuite('yt-dlp') - runSuite('youtube-dl') + + // FIXME: With recent changes on youtube, youtube-dl doesn't fetch live replays which means the test suite fails + // runSuite('youtube-dl') }) diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts index 865b25f04..91291524d 100644 --- a/server/tests/api/videos/video-channel-syncs.ts +++ b/server/tests/api/videos/video-channel-syncs.ts @@ -220,7 +220,7 @@ describe('Test channel synchronizations', function () { expect(total).to.equal(0) }) - // FIXME: youtube-dl doesn't work when speicifying a port after the hostname + // FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname // it('Should import a remote PeerTube channel', async function () { // this.timeout(240_000) -- cgit v1.2.3