aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-09-07 17:30:21 +0200
committerChocobozzz <me@florianbigard.com>2022-09-08 08:41:48 +0200
commite9fc9e03c120fb048ed00b38157d15144770ec23 (patch)
tree5bea5b738e518527663f17217c92c24fde6620f4
parentd4d9bbc6f24522f5d63b0ab105a02f80ca98d702 (diff)
downloadPeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.tar.gz
PeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.tar.zst
PeerTube-e9fc9e03c120fb048ed00b38157d15144770ec23.zip
Optimize fetching playlist urls
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts5
-rw-r--r--server/helpers/youtube-dl/youtube-dl-wrapper.ts9
-rw-r--r--server/lib/sync-channel.ts42
-rw-r--r--server/tests/api/videos/multiple-servers.ts2
4 files changed, 37 insertions, 21 deletions
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 508055b85..fc4c40787 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -142,6 +142,11 @@ export class YoutubeDLCLI {
142 }): Promise<{ upload_date: string, webpage_url: string }[]> { 142 }): Promise<{ upload_date: string, webpage_url: string }[]> {
143 const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ] 143 const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
144 144
145 if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
146 // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
147 additionalYoutubeDLArgs.push('--flat-playlist')
148 }
149
145 if (options.latestVideosCount !== undefined) { 150 if (options.latestVideosCount !== undefined) {
146 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) 151 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
147 } 152 }
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index 2c3ba2feb..966b8df78 100644
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -1,5 +1,6 @@
1import { move, pathExists, readdir, remove } from 'fs-extra' 1import { move, pathExists, readdir, remove } from 'fs-extra'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { inspect } from 'util'
3import { CONFIG } from '@server/initializers/config' 4import { CONFIG } from '@server/initializers/config'
4import { isVideoFileExtnameValid } from '../custom-validators/videos' 5import { isVideoFileExtnameValid } from '../custom-validators/videos'
5import { logger, loggerTagsFactory } from '../logger' 6import { logger, loggerTagsFactory } from '../logger'
@@ -59,13 +60,9 @@ class YoutubeDLWrapper {
59 processOptions 60 processOptions
60 }) 61 })
61 62
62 if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}`) 63 if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
63 64
64 return list.map(info => { 65 return list.map(info => info.webpage_url)
65 const infoBuilder = new YoutubeDLInfoBuilder(info)
66
67 return infoBuilder.getInfo()
68 })
69 } 66 }
70 67
71 async getSubtitles (): Promise<YoutubeDLSubs> { 68 async getSubtitles (): Promise<YoutubeDLSubs> {
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index eb5ca1703..3a81daac0 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
4import { buildYoutubeDLImport } from '@server/lib/video-import' 4import { buildYoutubeDLImport } from '@server/lib/video-import'
5import { UserModel } from '@server/models/user/user' 5import { UserModel } from '@server/models/user/user'
6import { VideoImportModel } from '@server/models/video/video-import' 6import { VideoImportModel } from '@server/models/video/video-import'
7import { MChannelAccountDefault, MChannelSync } from '@server/types/models' 7import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
8import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' 8import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
9import { CreateJobArgument, JobQueue } from './job-queue' 9import { CreateJobArgument, JobQueue } from './job-queue'
10import { ServerConfigManager } from './server-config-manager' 10import { ServerConfigManager } from './server-config-manager'
@@ -31,15 +31,7 @@ export async function synchronizeChannel (options: {
31 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION 31 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
32 ) 32 )
33 33
34 const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) 34 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
35
36 const targetUrls = infoList
37 .filter(videoInfo => {
38 if (!onlyAfter) return true
39
40 return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
41 })
42 .map(videoInfo => videoInfo.webpageUrl)
43 35
44 logger.info( 36 logger.info(
45 'Fetched %d candidate URLs for sync channel %s.', 37 'Fetched %d candidate URLs for sync channel %s.',
@@ -58,10 +50,7 @@ export async function synchronizeChannel (options: {
58 const children: CreateJobArgument[] = [] 50 const children: CreateJobArgument[] = []
59 51
60 for (const targetUrl of targetUrls) { 52 for (const targetUrl of targetUrls) {
61 if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { 53 if (await skipImport(channel, targetUrl, onlyAfter)) continue
62 logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
63 continue
64 }
65 54
66 const { job } = await buildYoutubeDLImport({ 55 const { job } = await buildYoutubeDLImport({
67 user, 56 user,
@@ -86,3 +75,28 @@ export async function synchronizeChannel (options: {
86 75
87 await JobQueue.Instance.createJobWithChildren(parent, children) 76 await JobQueue.Instance.createJobWithChildren(parent, children)
88} 77}
78
79// ---------------------------------------------------------------------------
80
81async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
82 if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
83 logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
84 return true
85 }
86
87 if (onlyAfter) {
88 const youtubeDL = new YoutubeDLWrapper(
89 targetUrl,
90 ServerConfigManager.Instance.getEnabledResolutions('vod'),
91 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
92 )
93
94 const videoInfo = await youtubeDL.getInfoForDownload()
95
96 if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
97 return true
98 }
99 }
100
101 return false
102}
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
index 23790ba65..d47807a79 100644
--- a/server/tests/api/videos/multiple-servers.ts
+++ b/server/tests/api/videos/multiple-servers.ts
@@ -1021,7 +1021,7 @@ describe('Test multiple servers', function () {
1021 1021
1022 describe('With minimum parameters', function () { 1022 describe('With minimum parameters', function () {
1023 it('Should upload and propagate the video', async function () { 1023 it('Should upload and propagate the video', async function () {
1024 this.timeout(60000) 1024 this.timeout(120000)
1025 1025
1026 const path = '/api/v1/videos/upload' 1026 const path = '/api/v1/videos/upload'
1027 1027