aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-02-09 11:40:47 +0100
committerChocobozzz <me@florianbigard.com>2022-02-09 11:54:18 +0100
commit7630e1c893f9848dce9d94135ce9b9a21ab80788 (patch)
treec5d171141b471d545c5da3b7ad27c4312dfd9b8d
parent474542d7ac60f7860daf9ea34d1c31968f43ab29 (diff)
downloadPeerTube-7630e1c893f9848dce9d94135ce9b9a21ab80788.tar.gz
PeerTube-7630e1c893f9848dce9d94135ce9b9a21ab80788.tar.zst
PeerTube-7630e1c893f9848dce9d94135ce9b9a21ab80788.zip
Fix import timeout inconsistency
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts13
-rw-r--r--server/helpers/youtube-dl/youtube-dl-wrapper.ts50
-rw-r--r--server/initializers/constants.ts2
-rw-r--r--server/lib/job-queue/handlers/video-import.ts6
4 files changed, 37 insertions, 34 deletions
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 293acff43..d4a6dd9b3 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -90,11 +90,13 @@ export class YoutubeDLCLI {
90 format: string 90 format: string
91 output: string 91 output: string
92 processOptions: execa.NodeOptions 92 processOptions: execa.NodeOptions
93 timeout: number
93 additionalYoutubeDLArgs?: string[] 94 additionalYoutubeDLArgs?: string[]
94 }) { 95 }) {
95 return this.run({ 96 return this.run({
96 url: options.url, 97 url: options.url,
97 processOptions: options.processOptions, 98 processOptions: options.processOptions,
99 timeout: options.timeout,
98 args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ]) 100 args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
99 }) 101 })
100 } 102 }
@@ -145,16 +147,23 @@ export class YoutubeDLCLI {
145 private async run (options: { 147 private async run (options: {
146 url: string 148 url: string
147 args: string[] 149 args: string[]
150 timeout?: number
148 processOptions: execa.NodeOptions 151 processOptions: execa.NodeOptions
149 }) { 152 }) {
150 const { url, args, processOptions } = options 153 const { url, args, timeout, processOptions } = options
151 154
152 let completeArgs = this.wrapWithProxyOptions(args) 155 let completeArgs = this.wrapWithProxyOptions(args)
153 completeArgs = this.wrapWithIPOptions(completeArgs) 156 completeArgs = this.wrapWithIPOptions(completeArgs)
154 completeArgs = this.wrapWithFFmpegOptions(completeArgs) 157 completeArgs = this.wrapWithFFmpegOptions(completeArgs)
155 158
156 const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE 159 const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE
157 const output = await execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) 160 const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
161
162 if (timeout) {
163 setTimeout(() => subProcess.cancel(), timeout)
164 }
165
166 const output = await subProcess
158 167
159 logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() }) 168 logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
160 169
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index 6442c1e85..68c29cd86 100644
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -77,38 +77,32 @@ class YoutubeDLWrapper {
77 77
78 const youtubeDL = await YoutubeDLCLI.safeGet() 78 const youtubeDL = await YoutubeDLCLI.safeGet()
79 79
80 let timer: NodeJS.Timeout 80 try {
81 const timeoutPromise = new Promise<string>((_, rej) => { 81 await youtubeDL.download({
82 timer = setTimeout(() => rej(new Error('YoutubeDL download timeout.')), timeout) 82 url: this.url,
83 }) 83 format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions),
84 84 output: pathWithoutExtension,
85 const downloadPromise = youtubeDL.download({ 85 timeout,
86 url: this.url, 86 processOptions
87 format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions),
88 output: pathWithoutExtension,
89 processOptions
90 }).then(() => clearTimeout(timer))
91 .then(async () => {
92 // If youtube-dl did not guess an extension for our file, just use .mp4 as default
93 if (await pathExists(pathWithoutExtension)) {
94 await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
95 }
96
97 return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
98 }) 87 })
99 88
100 return Promise.race([ downloadPromise, timeoutPromise ]) 89 // If youtube-dl did not guess an extension for our file, just use .mp4 as default
101 .catch(err => { 90 if (await pathExists(pathWithoutExtension)) {
102 this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) 91 await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
103 .then(path => { 92 }
104 logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })
105 93
106 return remove(path) 94 return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
107 }) 95 } catch (err) {
108 .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) 96 this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
97 .then(path => {
98 logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })
109 99
110 throw err 100 return remove(path)
111 }) 101 })
102 .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() }))
103
104 throw err
105 }
112 } 106 }
113 107
114 private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { 108 private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6a59bf805..f66e6b65d 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -215,7 +215,7 @@ const REQUEST_TIMEOUTS = {
215} 215}
216 216
217const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days 217const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
218const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour 218const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9)
219 219
220const SCHEDULER_INTERVALS_MS = { 220const SCHEDULER_INTERVALS_MS = {
221 ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour 221 ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 1630ecabd..b6e05d8f5 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -29,7 +29,7 @@ import { ffprobePromise, getDurationFromVideoFile, getVideoFileFPS, getVideoFile
29import { logger } from '../../../helpers/logger' 29import { logger } from '../../../helpers/logger'
30import { getSecureTorrentName } from '../../../helpers/utils' 30import { getSecureTorrentName } from '../../../helpers/utils'
31import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' 31import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
32import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' 32import { JOB_TTL } from '../../../initializers/constants'
33import { sequelizeTypescript } from '../../../initializers/database' 33import { sequelizeTypescript } from '../../../initializers/database'
34import { VideoModel } from '../../../models/video/video' 34import { VideoModel } from '../../../models/video/video'
35import { VideoFileModel } from '../../../models/video/video-file' 35import { VideoFileModel } from '../../../models/video/video-file'
@@ -72,7 +72,7 @@ async function processTorrentImport (job: Job, videoImport: MVideoImportDefault,
72 torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, 72 torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
73 uri: videoImport.magnetUri 73 uri: videoImport.magnetUri
74 } 74 }
75 return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) 75 return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options)
76} 76}
77 77
78async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) { 78async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) {
@@ -83,7 +83,7 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul
83 const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod')) 83 const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
84 84
85 return processFile( 85 return processFile(
86 () => youtubeDL.downloadVideo(payload.fileExt, VIDEO_IMPORT_TIMEOUT), 86 () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),
87 videoImport, 87 videoImport,
88 options 88 options
89 ) 89 )