diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/helpers/youtube-dl/youtube-dl-cli.ts | 13 | ||||
-rw-r--r-- | server/helpers/youtube-dl/youtube-dl-wrapper.ts | 50 | ||||
-rw-r--r-- | server/initializers/constants.ts | 2 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-import.ts | 6 |
4 files changed, 37 insertions, 34 deletions
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts index 293acff43..d4a6dd9b3 100644 --- a/server/helpers/youtube-dl/youtube-dl-cli.ts +++ b/server/helpers/youtube-dl/youtube-dl-cli.ts | |||
@@ -90,11 +90,13 @@ export class YoutubeDLCLI { | |||
90 | format: string | 90 | format: string |
91 | output: string | 91 | output: string |
92 | processOptions: execa.NodeOptions | 92 | processOptions: execa.NodeOptions |
93 | timeout: number | ||
93 | additionalYoutubeDLArgs?: string[] | 94 | additionalYoutubeDLArgs?: string[] |
94 | }) { | 95 | }) { |
95 | return this.run({ | 96 | return this.run({ |
96 | url: options.url, | 97 | url: options.url, |
97 | processOptions: options.processOptions, | 98 | processOptions: options.processOptions, |
99 | timeout: options.timeout, | ||
98 | args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ]) | 100 | args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ]) |
99 | }) | 101 | }) |
100 | } | 102 | } |
@@ -145,16 +147,23 @@ export class YoutubeDLCLI { | |||
145 | private async run (options: { | 147 | private async run (options: { |
146 | url: string | 148 | url: string |
147 | args: string[] | 149 | args: string[] |
150 | timeout?: number | ||
148 | processOptions: execa.NodeOptions | 151 | processOptions: execa.NodeOptions |
149 | }) { | 152 | }) { |
150 | const { url, args, processOptions } = options | 153 | const { url, args, timeout, processOptions } = options |
151 | 154 | ||
152 | let completeArgs = this.wrapWithProxyOptions(args) | 155 | let completeArgs = this.wrapWithProxyOptions(args) |
153 | completeArgs = this.wrapWithIPOptions(completeArgs) | 156 | completeArgs = this.wrapWithIPOptions(completeArgs) |
154 | completeArgs = this.wrapWithFFmpegOptions(completeArgs) | 157 | completeArgs = this.wrapWithFFmpegOptions(completeArgs) |
155 | 158 | ||
156 | const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE | 159 | const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE |
157 | const output = await execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) | 160 | const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) |
161 | |||
162 | if (timeout) { | ||
163 | setTimeout(() => subProcess.cancel(), timeout) | ||
164 | } | ||
165 | |||
166 | const output = await subProcess | ||
158 | 167 | ||
159 | logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() }) | 168 | logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() }) |
160 | 169 | ||
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts index 6442c1e85..68c29cd86 100644 --- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts +++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts | |||
@@ -77,38 +77,32 @@ class YoutubeDLWrapper { | |||
77 | 77 | ||
78 | const youtubeDL = await YoutubeDLCLI.safeGet() | 78 | const youtubeDL = await YoutubeDLCLI.safeGet() |
79 | 79 | ||
80 | let timer: NodeJS.Timeout | 80 | try { |
81 | const timeoutPromise = new Promise<string>((_, rej) => { | 81 | await youtubeDL.download({ |
82 | timer = setTimeout(() => rej(new Error('YoutubeDL download timeout.')), timeout) | 82 | url: this.url, |
83 | }) | 83 | format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions), |
84 | 84 | output: pathWithoutExtension, | |
85 | const downloadPromise = youtubeDL.download({ | 85 | timeout, |
86 | url: this.url, | 86 | processOptions |
87 | format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions), | ||
88 | output: pathWithoutExtension, | ||
89 | processOptions | ||
90 | }).then(() => clearTimeout(timer)) | ||
91 | .then(async () => { | ||
92 | // If youtube-dl did not guess an extension for our file, just use .mp4 as default | ||
93 | if (await pathExists(pathWithoutExtension)) { | ||
94 | await move(pathWithoutExtension, pathWithoutExtension + '.mp4') | ||
95 | } | ||
96 | |||
97 | return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) | ||
98 | }) | 87 | }) |
99 | 88 | ||
100 | return Promise.race([ downloadPromise, timeoutPromise ]) | 89 | // If youtube-dl did not guess an extension for our file, just use .mp4 as default |
101 | .catch(err => { | 90 | if (await pathExists(pathWithoutExtension)) { |
102 | this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) | 91 | await move(pathWithoutExtension, pathWithoutExtension + '.mp4') |
103 | .then(path => { | 92 | } |
104 | logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() }) | ||
105 | 93 | ||
106 | return remove(path) | 94 | return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) |
107 | }) | 95 | } catch (err) { |
108 | .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) | 96 | this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) |
97 | .then(path => { | ||
98 | logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() }) | ||
109 | 99 | ||
110 | throw err | 100 | return remove(path) |
111 | }) | 101 | }) |
102 | .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) | ||
103 | |||
104 | throw err | ||
105 | } | ||
112 | } | 106 | } |
113 | 107 | ||
114 | private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { | 108 | private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6a59bf805..f66e6b65d 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -215,7 +215,7 @@ const REQUEST_TIMEOUTS = { | |||
215 | } | 215 | } |
216 | 216 | ||
217 | const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days | 217 | const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days |
218 | const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour | 218 | const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9) |
219 | 219 | ||
220 | const SCHEDULER_INTERVALS_MS = { | 220 | const SCHEDULER_INTERVALS_MS = { |
221 | ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour | 221 | ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 1630ecabd..b6e05d8f5 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -29,7 +29,7 @@ import { ffprobePromise, getDurationFromVideoFile, getVideoFileFPS, getVideoFile | |||
29 | import { logger } from '../../../helpers/logger' | 29 | import { logger } from '../../../helpers/logger' |
30 | import { getSecureTorrentName } from '../../../helpers/utils' | 30 | import { getSecureTorrentName } from '../../../helpers/utils' |
31 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 31 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
32 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' | 32 | import { JOB_TTL } from '../../../initializers/constants' |
33 | import { sequelizeTypescript } from '../../../initializers/database' | 33 | import { sequelizeTypescript } from '../../../initializers/database' |
34 | import { VideoModel } from '../../../models/video/video' | 34 | import { VideoModel } from '../../../models/video/video' |
35 | import { VideoFileModel } from '../../../models/video/video-file' | 35 | import { VideoFileModel } from '../../../models/video/video-file' |
@@ -72,7 +72,7 @@ async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, | |||
72 | torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, | 72 | torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, |
73 | uri: videoImport.magnetUri | 73 | uri: videoImport.magnetUri |
74 | } | 74 | } |
75 | return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) | 75 | return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options) |
76 | } | 76 | } |
77 | 77 | ||
78 | async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) { | 78 | async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) { |
@@ -83,7 +83,7 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul | |||
83 | const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod')) | 83 | const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod')) |
84 | 84 | ||
85 | return processFile( | 85 | return processFile( |
86 | () => youtubeDL.downloadVideo(payload.fileExt, VIDEO_IMPORT_TIMEOUT), | 86 | () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']), |
87 | videoImport, | 87 | videoImport, |
88 | options | 88 | options |
89 | ) | 89 | ) |