diff options
-rw-r--r-- | server/lib/job-queue/handlers/video-channel-import.ts | 22 | ||||
-rw-r--r-- | server/lib/schedulers/video-channel-sync-latest-scheduler.ts | 35 | ||||
-rw-r--r-- | server/lib/sync-channel.ts | 90 |
3 files changed, 70 insertions, 77 deletions
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts index 600292844..c3dd8a688 100644 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ b/server/lib/job-queue/handlers/video-channel-import.ts | |||
@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel' | |||
5 | import { VideoChannelModel } from '@server/models/video/video-channel' | 5 | import { VideoChannelModel } from '@server/models/video/video-channel' |
6 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | 6 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' |
7 | import { MChannelSync } from '@server/types/models' | 7 | import { MChannelSync } from '@server/types/models' |
8 | import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models' | 8 | import { VideoChannelImportPayload } from '@shared/models' |
9 | 9 | ||
10 | export async function processVideoChannelImport (job: Job) { | 10 | export async function processVideoChannelImport (job: Job) { |
11 | const payload = job.data as VideoChannelImportPayload | 11 | const payload = job.data as VideoChannelImportPayload |
@@ -32,17 +32,11 @@ export async function processVideoChannelImport (job: Job) { | |||
32 | 32 | ||
33 | const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) | 33 | const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) |
34 | 34 | ||
35 | try { | 35 | logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) |
36 | logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) | 36 | |
37 | 37 | await synchronizeChannel({ | |
38 | await synchronizeChannel({ | 38 | channel: videoChannel, |
39 | channel: videoChannel, | 39 | externalChannelUrl: payload.externalChannelUrl, |
40 | externalChannelUrl: payload.externalChannelUrl, | 40 | channelSync |
41 | channelSync | 41 | }) |
42 | }) | ||
43 | } catch (err) { | ||
44 | logger.error(`Failed to import channel ${videoChannel.name}`, { err }) | ||
45 | channelSync.state = VideoChannelSyncState.FAILED | ||
46 | await channelSync.save() | ||
47 | } | ||
48 | } | 42 | } |
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts index a527f68b5..efb957fac 100644 --- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts +++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts | |||
@@ -2,7 +2,6 @@ import { logger } from '@server/helpers/logger' | |||
2 | import { CONFIG } from '@server/initializers/config' | 2 | import { CONFIG } from '@server/initializers/config' |
3 | import { VideoChannelModel } from '@server/models/video/video-channel' | 3 | import { VideoChannelModel } from '@server/models/video/video-channel' |
4 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | 4 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' |
5 | import { VideoChannelSyncState } from '@shared/models' | ||
6 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
7 | import { synchronizeChannel } from '../sync-channel' | 6 | import { synchronizeChannel } from '../sync-channel' |
8 | import { AbstractScheduler } from './abstract-scheduler' | 7 | import { AbstractScheduler } from './abstract-scheduler' |
@@ -28,26 +27,20 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler { | |||
28 | for (const sync of channelSyncs) { | 27 | for (const sync of channelSyncs) { |
29 | const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) | 28 | const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) |
30 | 29 | ||
31 | try { | 30 | logger.info( |
32 | logger.info( | 31 | 'Creating video import jobs for "%s" sync with external channel "%s"', |
33 | 'Creating video import jobs for "%s" sync with external channel "%s"', | 32 | channel.Actor.preferredUsername, sync.externalChannelUrl |
34 | channel.Actor.preferredUsername, sync.externalChannelUrl | 33 | ) |
35 | ) | 34 | |
36 | 35 | const onlyAfter = sync.lastSyncAt || sync.createdAt | |
37 | const onlyAfter = sync.lastSyncAt || sync.createdAt | 36 | |
38 | 37 | await synchronizeChannel({ | |
39 | await synchronizeChannel({ | 38 | channel, |
40 | channel, | 39 | externalChannelUrl: sync.externalChannelUrl, |
41 | externalChannelUrl: sync.externalChannelUrl, | 40 | videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, |
42 | videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, | 41 | channelSync: sync, |
43 | channelSync: sync, | 42 | onlyAfter |
44 | onlyAfter | 43 | }) |
45 | }) | ||
46 | } catch (err) { | ||
47 | logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err }) | ||
48 | sync.state = VideoChannelSyncState.FAILED | ||
49 | await sync.save() | ||
50 | } | ||
51 | } | 44 | } |
52 | } | 45 | } |
53 | 46 | ||
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts index f91599c14..35af91429 100644 --- a/server/lib/sync-channel.ts +++ b/server/lib/sync-channel.ts | |||
@@ -24,56 +24,62 @@ export async function synchronizeChannel (options: { | |||
24 | await channelSync.save() | 24 | await channelSync.save() |
25 | } | 25 | } |
26 | 26 | ||
27 | const user = await UserModel.loadByChannelActorId(channel.actorId) | 27 | try { |
28 | const youtubeDL = new YoutubeDLWrapper( | 28 | const user = await UserModel.loadByChannelActorId(channel.actorId) |
29 | externalChannelUrl, | 29 | const youtubeDL = new YoutubeDLWrapper( |
30 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | 30 | externalChannelUrl, |
31 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | 31 | ServerConfigManager.Instance.getEnabledResolutions('vod'), |
32 | ) | 32 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION |
33 | 33 | ) | |
34 | const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) | ||
35 | |||
36 | logger.info( | ||
37 | 'Fetched %d candidate URLs for sync channel %s.', | ||
38 | targetUrls.length, channel.Actor.preferredUsername, { targetUrls } | ||
39 | ) | ||
40 | |||
41 | if (targetUrls.length === 0) { | ||
42 | if (channelSync) { | ||
43 | channelSync.state = VideoChannelSyncState.SYNCED | ||
44 | await channelSync.save() | ||
45 | } | ||
46 | |||
47 | return | ||
48 | } | ||
49 | 34 | ||
50 | const children: CreateJobArgument[] = [] | 35 | const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) |
51 | 36 | ||
52 | for (const targetUrl of targetUrls) { | 37 | logger.info( |
53 | if (await skipImport(channel, targetUrl, onlyAfter)) continue | 38 | 'Fetched %d candidate URLs for sync channel %s.', |
39 | targetUrls.length, channel.Actor.preferredUsername, { targetUrls } | ||
40 | ) | ||
54 | 41 | ||
55 | const { job } = await buildYoutubeDLImport({ | 42 | if (targetUrls.length === 0) { |
56 | user, | 43 | if (channelSync) { |
57 | channel, | 44 | channelSync.state = VideoChannelSyncState.SYNCED |
58 | targetUrl, | 45 | await channelSync.save() |
59 | channelSync, | ||
60 | importDataOverride: { | ||
61 | privacy: VideoPrivacy.PUBLIC | ||
62 | } | 46 | } |
63 | }) | ||
64 | 47 | ||
65 | children.push(job) | 48 | return |
66 | } | 49 | } |
50 | |||
51 | const children: CreateJobArgument[] = [] | ||
52 | |||
53 | for (const targetUrl of targetUrls) { | ||
54 | if (await skipImport(channel, targetUrl, onlyAfter)) continue | ||
67 | 55 | ||
68 | // Will update the channel sync status | 56 | const { job } = await buildYoutubeDLImport({ |
69 | const parent: CreateJobArgument = { | 57 | user, |
70 | type: 'after-video-channel-import', | 58 | channel, |
71 | payload: { | 59 | targetUrl, |
72 | channelSyncId: channelSync?.id | 60 | channelSync, |
61 | importDataOverride: { | ||
62 | privacy: VideoPrivacy.PUBLIC | ||
63 | } | ||
64 | }) | ||
65 | |||
66 | children.push(job) | ||
73 | } | 67 | } |
74 | } | ||
75 | 68 | ||
76 | await JobQueue.Instance.createJobWithChildren(parent, children) | 69 | // Will update the channel sync status |
70 | const parent: CreateJobArgument = { | ||
71 | type: 'after-video-channel-import', | ||
72 | payload: { | ||
73 | channelSyncId: channelSync?.id | ||
74 | } | ||
75 | } | ||
76 | |||
77 | await JobQueue.Instance.createJobWithChildren(parent, children) | ||
78 | } catch (err) { | ||
79 | logger.error(`Failed to import channel ${channel.name}`, { err }) | ||
80 | channelSync.state = VideoChannelSyncState.FAILED | ||
81 | await channelSync.save() | ||
82 | } | ||
77 | } | 83 | } |
78 | 84 | ||
79 | // --------------------------------------------------------------------------- | 85 | // --------------------------------------------------------------------------- |