diff options
author | Chocobozzz <me@florianbigard.com> | 2022-09-07 17:30:21 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2022-09-08 08:41:48 +0200 |
commit | e9fc9e03c120fb048ed00b38157d15144770ec23 (patch) | |
tree | 5bea5b738e518527663f17217c92c24fde6620f4 /server/lib | |
parent | d4d9bbc6f24522f5d63b0ab105a02f80ca98d702 (diff) | |
download | PeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.tar.gz PeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.tar.zst PeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.zip |
Optimize fetching playlist urls
Diffstat (limited to 'server/lib')
-rw-r--r-- | server/lib/sync-channel.ts | 42 |
1 files changed, 28 insertions, 14 deletions
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts index eb5ca1703..3a81daac0 100644 --- a/server/lib/sync-channel.ts +++ b/server/lib/sync-channel.ts | |||
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config' | |||
4 | import { buildYoutubeDLImport } from '@server/lib/video-import' | 4 | import { buildYoutubeDLImport } from '@server/lib/video-import' |
5 | import { UserModel } from '@server/models/user/user' | 5 | import { UserModel } from '@server/models/user/user' |
6 | import { VideoImportModel } from '@server/models/video/video-import' | 6 | import { VideoImportModel } from '@server/models/video/video-import' |
7 | import { MChannelAccountDefault, MChannelSync } from '@server/types/models' | 7 | import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models' |
8 | import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' | 8 | import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' |
9 | import { CreateJobArgument, JobQueue } from './job-queue' | 9 | import { CreateJobArgument, JobQueue } from './job-queue' |
10 | import { ServerConfigManager } from './server-config-manager' | 10 | import { ServerConfigManager } from './server-config-manager' |
@@ -31,15 +31,7 @@ export async function synchronizeChannel (options: { | |||
31 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | 31 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION |
32 | ) | 32 | ) |
33 | 33 | ||
34 | const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) | 34 | const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) |
35 | |||
36 | const targetUrls = infoList | ||
37 | .filter(videoInfo => { | ||
38 | if (!onlyAfter) return true | ||
39 | |||
40 | return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime() | ||
41 | }) | ||
42 | .map(videoInfo => videoInfo.webpageUrl) | ||
43 | 35 | ||
44 | logger.info( | 36 | logger.info( |
45 | 'Fetched %d candidate URLs for sync channel %s.', | 37 | 'Fetched %d candidate URLs for sync channel %s.', |
@@ -58,10 +50,7 @@ export async function synchronizeChannel (options: { | |||
58 | const children: CreateJobArgument[] = [] | 50 | const children: CreateJobArgument[] = [] |
59 | 51 | ||
60 | for (const targetUrl of targetUrls) { | 52 | for (const targetUrl of targetUrls) { |
61 | if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { | 53 | if (await skipImport(channel, targetUrl, onlyAfter)) continue |
62 | logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl) | ||
63 | continue | ||
64 | } | ||
65 | 54 | ||
66 | const { job } = await buildYoutubeDLImport({ | 55 | const { job } = await buildYoutubeDLImport({ |
67 | user, | 56 | user, |
@@ -86,3 +75,28 @@ export async function synchronizeChannel (options: { | |||
86 | 75 | ||
87 | await JobQueue.Instance.createJobWithChildren(parent, children) | 76 | await JobQueue.Instance.createJobWithChildren(parent, children) |
88 | } | 77 | } |
78 | |||
79 | // --------------------------------------------------------------------------- | ||
80 | |||
81 | async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) { | ||
82 | if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { | ||
83 | logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl) | ||
84 | return true | ||
85 | } | ||
86 | |||
87 | if (onlyAfter) { | ||
88 | const youtubeDL = new YoutubeDLWrapper( | ||
89 | targetUrl, | ||
90 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | ||
91 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
92 | ) | ||
93 | |||
94 | const videoInfo = await youtubeDL.getInfoForDownload() | ||
95 | |||
96 | if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) { | ||
97 | return true | ||
98 | } | ||
99 | } | ||
100 | |||
101 | return false | ||
102 | } | ||