}): Promise<{ upload_date: string, webpage_url: string }[]> {
const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
+ if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
+ // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
+ additionalYoutubeDLArgs.push('--flat-playlist')
+ }
+
if (options.latestVideosCount !== undefined) {
additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
}
import { move, pathExists, readdir, remove } from 'fs-extra'
import { dirname, join } from 'path'
+import { inspect } from 'util'
import { CONFIG } from '@server/initializers/config'
import { isVideoFileExtnameValid } from '../custom-validators/videos'
import { logger, loggerTagsFactory } from '../logger'
processOptions
})
- if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}`)
+ if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
- return list.map(info => {
- const infoBuilder = new YoutubeDLInfoBuilder(info)
-
- return infoBuilder.getInfo()
- })
+ return list.map(info => info.webpage_url)
}
async getSubtitles (): Promise<YoutubeDLSubs> {
import { buildYoutubeDLImport } from '@server/lib/video-import'
import { UserModel } from '@server/models/user/user'
import { VideoImportModel } from '@server/models/video/video-import'
-import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
+import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
import { CreateJobArgument, JobQueue } from './job-queue'
import { ServerConfigManager } from './server-config-manager'
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
- const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
-
- const targetUrls = infoList
- .filter(videoInfo => {
- if (!onlyAfter) return true
-
- return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
- })
- .map(videoInfo => videoInfo.webpageUrl)
+ const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
logger.info(
'Fetched %d candidate URLs for sync channel %s.',
const children: CreateJobArgument[] = []
for (const targetUrl of targetUrls) {
- if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
- logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
- continue
- }
+ if (await skipImport(channel, targetUrl, onlyAfter)) continue
const { job } = await buildYoutubeDLImport({
user,
await JobQueue.Instance.createJobWithChildren(parent, children)
}
+
+// ---------------------------------------------------------------------------
+
+async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
+ if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
+ logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
+ return true
+ }
+
+ if (onlyAfter) {
+ const youtubeDL = new YoutubeDLWrapper(
+ targetUrl,
+ ServerConfigManager.Instance.getEnabledResolutions('vod'),
+ CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+ )
+
+ const videoInfo = await youtubeDL.getInfoForDownload()
+
+ if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
+ return true
+ }
+ }
+
+ return false
+}