From 7630e1c893f9848dce9d94135ce9b9a21ab80788 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 9 Feb 2022 11:40:47 +0100 Subject: Fix import timeout inconsistency --- server/helpers/youtube-dl/youtube-dl-cli.ts | 13 ++++++- server/helpers/youtube-dl/youtube-dl-wrapper.ts | 50 +++++++++++-------------- server/initializers/constants.ts | 2 +- server/lib/job-queue/handlers/video-import.ts | 6 +-- 4 files changed, 37 insertions(+), 34 deletions(-) (limited to 'server') diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts index 293acff43..d4a6dd9b3 100644 --- a/server/helpers/youtube-dl/youtube-dl-cli.ts +++ b/server/helpers/youtube-dl/youtube-dl-cli.ts @@ -90,11 +90,13 @@ export class YoutubeDLCLI { format: string output: string processOptions: execa.NodeOptions + timeout: number additionalYoutubeDLArgs?: string[] }) { return this.run({ url: options.url, processOptions: options.processOptions, + timeout: options.timeout, args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ]) }) } @@ -145,16 +147,23 @@ export class YoutubeDLCLI { private async run (options: { url: string args: string[] + timeout?: number processOptions: execa.NodeOptions }) { - const { url, args, processOptions } = options + const { url, args, timeout, processOptions } = options let completeArgs = this.wrapWithProxyOptions(args) completeArgs = this.wrapWithIPOptions(completeArgs) completeArgs = this.wrapWithFFmpegOptions(completeArgs) const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE - const output = await execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) + const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) + + if (timeout) { + setTimeout(() => subProcess.cancel(), timeout) + } + + const output = await subProcess logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() }) diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts index 6442c1e85..68c29cd86 100644 --- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts +++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts @@ -77,38 +77,32 @@ class YoutubeDLWrapper { const youtubeDL = await YoutubeDLCLI.safeGet() - let timer: NodeJS.Timeout - const timeoutPromise = new Promise((_, rej) => { - timer = setTimeout(() => rej(new Error('YoutubeDL download timeout.')), timeout) - }) - - const downloadPromise = youtubeDL.download({ - url: this.url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions), - output: pathWithoutExtension, - processOptions - }).then(() => clearTimeout(timer)) - .then(async () => { - // If youtube-dl did not guess an extension for our file, just use .mp4 as default - if (await pathExists(pathWithoutExtension)) { - await move(pathWithoutExtension, pathWithoutExtension + '.mp4') - } - - return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) + try { + await youtubeDL.download({ + url: this.url, + format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions), + output: pathWithoutExtension, + timeout, + processOptions }) - return Promise.race([ downloadPromise, timeoutPromise ]) - .catch(err => { - this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) - .then(path => { - logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() }) + // If youtube-dl did not guess an extension for our file, just use .mp4 as default + if (await pathExists(pathWithoutExtension)) { + await move(pathWithoutExtension, pathWithoutExtension + '.mp4') + } - return remove(path) - }) - .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) + return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) + } catch (err) { + this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) + .then(path => { + logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() }) - throw err - }) + return remove(path) + }) + .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) + + throw err + } } private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6a59bf805..f66e6b65d 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -215,7 +215,7 @@ const REQUEST_TIMEOUTS = { } const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days -const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour +const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9) const SCHEDULER_INTERVALS_MS = { ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 1630ecabd..b6e05d8f5 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -29,7 +29,7 @@ import { ffprobePromise, getDurationFromVideoFile, getVideoFileFPS, getVideoFile import { logger } from '../../../helpers/logger' import { getSecureTorrentName } from '../../../helpers/utils' import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' -import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' +import { JOB_TTL } from '../../../initializers/constants' import { sequelizeTypescript } from '../../../initializers/database' import { VideoModel } from '../../../models/video/video' import { VideoFileModel } from '../../../models/video/video-file' @@ -72,7 +72,7 @@ async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, uri: videoImport.magnetUri } - return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) + return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options) } async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) { @@ -83,7 +83,7 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod')) return processFile( - () => youtubeDL.downloadVideo(payload.fileExt, VIDEO_IMPORT_TIMEOUT), + () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']), videoImport, options ) -- cgit v1.2.3