diff options
-rw-r--r-- | server/controllers/api/videos/import.ts | 3 | ||||
-rw-r--r-- | server/helpers/utils.ts | 16 | ||||
-rw-r--r-- | server/helpers/youtube-dl.ts | 81 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-import.ts | 2 | ||||
-rw-r--r-- | shared/models/server/job.model.ts | 2 |
5 files changed, 52 insertions, 52 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 9702e219a..96499dffb 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -220,8 +220,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) | |||
220 | videoImportId: videoImport.id, | 220 | videoImportId: videoImport.id, |
221 | generateThumbnail: !thumbnailModel, | 221 | generateThumbnail: !thumbnailModel, |
222 | generatePreview: !previewModel, | 222 | generatePreview: !previewModel, |
223 | fileExt: `.${youtubeDLInfo.ext || 'mp4'}`, | 223 | fileExt: `.${youtubeDLInfo.ext || 'mp4'}` |
224 | mergeExt: youtubeDLInfo.mergeExt ? `.${youtubeDLInfo.mergeExt}` : '' | ||
225 | } | 224 | } |
226 | await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload }) | 225 | await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload }) |
227 | 226 | ||
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index ad3b7949e..0545e8996 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts | |||
@@ -1,11 +1,10 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { Instance as ParseTorrent } from 'parse-torrent' | ||
3 | import { join } from 'path' | ||
1 | import { ResultList } from '../../shared' | 4 | import { ResultList } from '../../shared' |
5 | import { CONFIG } from '../initializers/config' | ||
2 | import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' | 6 | import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' |
3 | import { logger } from './logger' | 7 | import { logger } from './logger' |
4 | import { join } from 'path' | ||
5 | import { Instance as ParseTorrent } from 'parse-torrent' | ||
6 | import { remove } from 'fs-extra' | ||
7 | import { CONFIG } from '../initializers/config' | ||
8 | import { isVideoFileExtnameValid } from './custom-validators/videos' | ||
9 | 8 | ||
10 | function deleteFileAsync (path: string) { | 9 | function deleteFileAsync (path: string) { |
11 | remove(path) | 10 | remove(path) |
@@ -31,16 +30,11 @@ function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: | |||
31 | } as ResultList<V> | 30 | } as ResultList<V> |
32 | } | 31 | } |
33 | 32 | ||
34 | function generateVideoImportTmpPath (target: string | ParseTorrent, extensionArg?: string) { | 33 | function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') { |
35 | const id = typeof target === 'string' | 34 | const id = typeof target === 'string' |
36 | ? target | 35 | ? target |
37 | : target.infoHash | 36 | : target.infoHash |
38 | 37 | ||
39 | let extension = '.mp4' | ||
40 | if (extensionArg && isVideoFileExtnameValid(extensionArg)) { | ||
41 | extension = extensionArg | ||
42 | } | ||
43 | |||
44 | const hash = sha256(id) | 38 | const hash = sha256(id) |
45 | return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`) | 39 | return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`) |
46 | } | 40 | } |
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 72d457bc1..6b9d8a5f7 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts | |||
@@ -1,15 +1,16 @@ | |||
1 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' | 1 | import { createWriteStream } from 'fs' |
2 | import { logger } from './logger' | 2 | import { ensureDir, pathExists, remove, writeFile } from 'fs-extra' |
3 | import { generateVideoImportTmpPath } from './utils' | ||
4 | import { getEnabledResolutions } from '../lib/video-transcoding' | ||
5 | import { join } from 'path' | 3 | import { join } from 'path' |
6 | import { peertubeTruncate, root } from './core-utils' | ||
7 | import { ensureDir, remove, writeFile } from 'fs-extra' | ||
8 | import * as request from 'request' | 4 | import * as request from 'request' |
9 | import { createWriteStream } from 'fs' | ||
10 | import { CONFIG } from '@server/initializers/config' | 5 | import { CONFIG } from '@server/initializers/config' |
11 | import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' | 6 | import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' |
12 | import { VideoResolution } from '../../shared/models/videos' | 7 | import { VideoResolution } from '../../shared/models/videos' |
8 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' | ||
9 | import { getEnabledResolutions } from '../lib/video-transcoding' | ||
10 | import { peertubeTruncate, root } from './core-utils' | ||
11 | import { isVideoFileExtnameValid } from './custom-validators/videos' | ||
12 | import { logger } from './logger' | ||
13 | import { generateVideoImportTmpPath } from './utils' | ||
13 | 14 | ||
14 | export type YoutubeDLInfo = { | 15 | export type YoutubeDLInfo = { |
15 | name?: string | 16 | name?: string |
@@ -21,7 +22,6 @@ export type YoutubeDLInfo = { | |||
21 | tags?: string[] | 22 | tags?: string[] |
22 | thumbnailUrl?: string | 23 | thumbnailUrl?: string |
23 | ext?: string | 24 | ext?: string |
24 | mergeExt?: string | ||
25 | originallyPublishedAt?: Date | 25 | originallyPublishedAt?: Date |
26 | } | 26 | } |
27 | 27 | ||
@@ -51,16 +51,6 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> | |||
51 | youtubeDL.getInfo(url, args, processOptions, (err, info) => { | 51 | youtubeDL.getInfo(url, args, processOptions, (err, info) => { |
52 | if (err) return rej(err) | 52 | if (err) return rej(err) |
53 | if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) | 53 | if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) |
54 | if (info.format_id?.includes('+')) { | ||
55 | // this is a merge format and its extension will be appended | ||
56 | if (info.ext === 'mp4') { | ||
57 | info.mergeExt = 'mp4' | ||
58 | } else if (info.ext === 'webm') { | ||
59 | info.mergeExt = 'webm' | ||
60 | } else { | ||
61 | info.mergeExt = 'mkv' | ||
62 | } | ||
63 | } | ||
64 | 54 | ||
65 | const obj = buildVideoInfo(normalizeObject(info)) | 55 | const obj = buildVideoInfo(normalizeObject(info)) |
66 | if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' | 56 | if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' |
@@ -133,18 +123,15 @@ function getYoutubeDLVideoFormat () { | |||
133 | ].join('/') | 123 | ].join('/') |
134 | } | 124 | } |
135 | 125 | ||
136 | function downloadYoutubeDLVideo (url: string, extension: string, timeout: number, mergeExtension?: string) { | 126 | function downloadYoutubeDLVideo (url: string, fileExt: string, timeout: number) { |
137 | let path = generateVideoImportTmpPath(url, extension) | 127 | // Leave empty the extension, youtube-dl will add it |
138 | 128 | const pathWithoutExtension = generateVideoImportTmpPath(url, '') | |
139 | path = mergeExtension | ||
140 | ? path.replace(new RegExp(`${extension}$`), mergeExtension) | ||
141 | : path | ||
142 | 129 | ||
143 | let timer | 130 | let timer |
144 | 131 | ||
145 | logger.info('Importing youtubeDL video %s to %s', url, path) | 132 | logger.info('Importing youtubeDL video %s to %s', url, pathWithoutExtension) |
146 | 133 | ||
147 | let options = [ '-f', getYoutubeDLVideoFormat(), '-o', path ] | 134 | let options = [ '-f', getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ] |
148 | options = wrapWithProxyOptions(options) | 135 | options = wrapWithProxyOptions(options) |
149 | 136 | ||
150 | if (process.env.FFMPEG_PATH) { | 137 | if (process.env.FFMPEG_PATH) { |
@@ -156,26 +143,33 @@ function downloadYoutubeDLVideo (url: string, extension: string, timeout: number | |||
156 | return new Promise<string>((res, rej) => { | 143 | return new Promise<string>((res, rej) => { |
157 | safeGetYoutubeDL() | 144 | safeGetYoutubeDL() |
158 | .then(youtubeDL => { | 145 | .then(youtubeDL => { |
159 | youtubeDL.exec(url, options, processOptions, err => { | 146 | youtubeDL.exec(url, options, processOptions, async err => { |
160 | clearTimeout(timer) | 147 | clearTimeout(timer) |
161 | 148 | ||
162 | if (err) { | 149 | try { |
163 | remove(path) | 150 | const path = await guessVideoPathWithExtension(pathWithoutExtension, fileExt) |
164 | .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) | 151 | |
152 | if (err) { | ||
153 | remove(path) | ||
154 | .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) | ||
165 | 155 | ||
156 | return rej(err) | ||
157 | } | ||
158 | |||
159 | return res(path) | ||
160 | } catch (err) { | ||
166 | return rej(err) | 161 | return rej(err) |
167 | } | 162 | } |
168 | |||
169 | return res(path) | ||
170 | }) | 163 | }) |
171 | 164 | ||
172 | timer = setTimeout(() => { | 165 | timer = setTimeout(() => { |
173 | const err = new Error('YoutubeDL download timeout.') | 166 | const err = new Error('YoutubeDL download timeout.') |
174 | 167 | ||
175 | remove(path) | 168 | guessVideoPathWithExtension(pathWithoutExtension, fileExt) |
169 | .then(path => remove(path)) | ||
176 | .finally(() => rej(err)) | 170 | .finally(() => rej(err)) |
177 | .catch(err => { | 171 | .catch(err => { |
178 | logger.error('Cannot remove %s in youtubeDL timeout.', path, { err }) | 172 | logger.error('Cannot remove file in youtubeDL timeout.', { err }) |
179 | return rej(err) | 173 | return rej(err) |
180 | }) | 174 | }) |
181 | }, timeout) | 175 | }, timeout) |
@@ -294,6 +288,22 @@ export { | |||
294 | 288 | ||
295 | // --------------------------------------------------------------------------- | 289 | // --------------------------------------------------------------------------- |
296 | 290 | ||
291 | async function guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { | ||
292 | if (!isVideoFileExtnameValid(sourceExt)) { | ||
293 | throw new Error('Invalid video extension ' + sourceExt) | ||
294 | } | ||
295 | |||
296 | const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ] | ||
297 | |||
298 | for (const extension of extensions) { | ||
299 | const path = tmpPath + extension | ||
300 | |||
301 | if (await pathExists(path)) return path | ||
302 | } | ||
303 | |||
304 | throw new Error('Cannot guess path of ' + tmpPath) | ||
305 | } | ||
306 | |||
297 | function normalizeObject (obj: any) { | 307 | function normalizeObject (obj: any) { |
298 | const newObj: any = {} | 308 | const newObj: any = {} |
299 | 309 | ||
@@ -324,8 +334,7 @@ function buildVideoInfo (obj: any): YoutubeDLInfo { | |||
324 | tags: getTags(obj.tags), | 334 | tags: getTags(obj.tags), |
325 | thumbnailUrl: obj.thumbnail || undefined, | 335 | thumbnailUrl: obj.thumbnail || undefined, |
326 | originallyPublishedAt: buildOriginallyPublishedAt(obj), | 336 | originallyPublishedAt: buildOriginallyPublishedAt(obj), |
327 | ext: obj.ext, | 337 | ext: obj.ext |
328 | mergeExt: obj.mergeExt | ||
329 | } | 338 | } |
330 | } | 339 | } |
331 | 340 | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index db3112418..c1d1866b0 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -80,7 +80,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub | |||
80 | } | 80 | } |
81 | 81 | ||
82 | return processFile( | 82 | return processFile( |
83 | () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT, payload.mergeExt), | 83 | () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), |
84 | videoImport, | 84 | videoImport, |
85 | options | 85 | options |
86 | ) | 86 | ) |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 2af2a25a6..f9a6250c9 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -80,9 +80,7 @@ export type VideoImportYoutubeDLPayload = { | |||
80 | 80 | ||
81 | generateThumbnail: boolean | 81 | generateThumbnail: boolean |
82 | generatePreview: boolean | 82 | generatePreview: boolean |
83 | |||
84 | fileExt?: string | 83 | fileExt?: string |
85 | mergeExt?: string | ||
86 | } | 84 | } |
87 | export type VideoImportTorrentPayload = { | 85 | export type VideoImportTorrentPayload = { |
88 | type: VideoImportTorrentPayloadType | 86 | type: VideoImportTorrentPayloadType |