aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-09-07 17:30:21 +0200
committerChocobozzz <me@florianbigard.com>2022-09-08 08:41:48 +0200
commite9fc9e03c120fb048ed00b38157d15144770ec23 (patch)
tree5bea5b738e518527663f17217c92c24fde6620f4 /server/lib
parentd4d9bbc6f24522f5d63b0ab105a02f80ca98d702 (diff)
downloadPeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.tar.gz
PeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.tar.zst
PeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.zip
Optimize fetching playlist urls
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/sync-channel.ts42
1 files changed, 28 insertions, 14 deletions
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index eb5ca1703..3a81daac0 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
4import { buildYoutubeDLImport } from '@server/lib/video-import' 4import { buildYoutubeDLImport } from '@server/lib/video-import'
5import { UserModel } from '@server/models/user/user' 5import { UserModel } from '@server/models/user/user'
6import { VideoImportModel } from '@server/models/video/video-import' 6import { VideoImportModel } from '@server/models/video/video-import'
7import { MChannelAccountDefault, MChannelSync } from '@server/types/models' 7import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
8import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' 8import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
9import { CreateJobArgument, JobQueue } from './job-queue' 9import { CreateJobArgument, JobQueue } from './job-queue'
10import { ServerConfigManager } from './server-config-manager' 10import { ServerConfigManager } from './server-config-manager'
@@ -31,15 +31,7 @@ export async function synchronizeChannel (options: {
31 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION 31 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
32 ) 32 )
33 33
34 const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) 34 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
35
36 const targetUrls = infoList
37 .filter(videoInfo => {
38 if (!onlyAfter) return true
39
40 return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
41 })
42 .map(videoInfo => videoInfo.webpageUrl)
43 35
44 logger.info( 36 logger.info(
45 'Fetched %d candidate URLs for sync channel %s.', 37 'Fetched %d candidate URLs for sync channel %s.',
@@ -58,10 +50,7 @@ export async function synchronizeChannel (options: {
58 const children: CreateJobArgument[] = [] 50 const children: CreateJobArgument[] = []
59 51
60 for (const targetUrl of targetUrls) { 52 for (const targetUrl of targetUrls) {
61 if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { 53 if (await skipImport(channel, targetUrl, onlyAfter)) continue
62 logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
63 continue
64 }
65 54
66 const { job } = await buildYoutubeDLImport({ 55 const { job } = await buildYoutubeDLImport({
67 user, 56 user,
@@ -86,3 +75,28 @@ export async function synchronizeChannel (options: {
86 75
87 await JobQueue.Instance.createJobWithChildren(parent, children) 76 await JobQueue.Instance.createJobWithChildren(parent, children)
88} 77}
78
79// ---------------------------------------------------------------------------
80
81async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
82 if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
83 logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
84 return true
85 }
86
87 if (onlyAfter) {
88 const youtubeDL = new YoutubeDLWrapper(
89 targetUrl,
90 ServerConfigManager.Instance.getEnabledResolutions('vod'),
91 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
92 )
93
94 const videoInfo = await youtubeDL.getInfoForDownload()
95
96 if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
97 return true
98 }
99 }
100
101 return false
102}