aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--server/controllers/api/videos/import.ts3
-rw-r--r--server/helpers/utils.ts16
-rw-r--r--server/helpers/youtube-dl.ts81
-rw-r--r--server/lib/job-queue/handlers/video-import.ts2
-rw-r--r--shared/models/server/job.model.ts2
5 files changed, 52 insertions, 52 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 9702e219a..96499dffb 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -220,8 +220,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
220 videoImportId: videoImport.id, 220 videoImportId: videoImport.id,
221 generateThumbnail: !thumbnailModel, 221 generateThumbnail: !thumbnailModel,
222 generatePreview: !previewModel, 222 generatePreview: !previewModel,
223 fileExt: `.${youtubeDLInfo.ext || 'mp4'}`, 223 fileExt: `.${youtubeDLInfo.ext || 'mp4'}`
224 mergeExt: youtubeDLInfo.mergeExt ? `.${youtubeDLInfo.mergeExt}` : ''
225 } 224 }
226 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload }) 225 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
227 226
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index ad3b7949e..0545e8996 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -1,11 +1,10 @@
1import { remove } from 'fs-extra'
2import { Instance as ParseTorrent } from 'parse-torrent'
3import { join } from 'path'
1import { ResultList } from '../../shared' 4import { ResultList } from '../../shared'
5import { CONFIG } from '../initializers/config'
2import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' 6import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
3import { logger } from './logger' 7import { logger } from './logger'
4import { join } from 'path'
5import { Instance as ParseTorrent } from 'parse-torrent'
6import { remove } from 'fs-extra'
7import { CONFIG } from '../initializers/config'
8import { isVideoFileExtnameValid } from './custom-validators/videos'
9 8
10function deleteFileAsync (path: string) { 9function deleteFileAsync (path: string) {
11 remove(path) 10 remove(path)
@@ -31,16 +30,11 @@ function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects:
31 } as ResultList<V> 30 } as ResultList<V>
32} 31}
33 32
34function generateVideoImportTmpPath (target: string | ParseTorrent, extensionArg?: string) { 33function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') {
35 const id = typeof target === 'string' 34 const id = typeof target === 'string'
36 ? target 35 ? target
37 : target.infoHash 36 : target.infoHash
38 37
39 let extension = '.mp4'
40 if (extensionArg && isVideoFileExtnameValid(extensionArg)) {
41 extension = extensionArg
42 }
43
44 const hash = sha256(id) 38 const hash = sha256(id)
45 return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`) 39 return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
46} 40}
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 72d457bc1..6b9d8a5f7 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,15 +1,16 @@
1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' 1import { createWriteStream } from 'fs'
2import { logger } from './logger' 2import { ensureDir, pathExists, remove, writeFile } from 'fs-extra'
3import { generateVideoImportTmpPath } from './utils'
4import { getEnabledResolutions } from '../lib/video-transcoding'
5import { join } from 'path' 3import { join } from 'path'
6import { peertubeTruncate, root } from './core-utils'
7import { ensureDir, remove, writeFile } from 'fs-extra'
8import * as request from 'request' 4import * as request from 'request'
9import { createWriteStream } from 'fs'
10import { CONFIG } from '@server/initializers/config' 5import { CONFIG } from '@server/initializers/config'
11import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' 6import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
12import { VideoResolution } from '../../shared/models/videos' 7import { VideoResolution } from '../../shared/models/videos'
8import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
9import { getEnabledResolutions } from '../lib/video-transcoding'
10import { peertubeTruncate, root } from './core-utils'
11import { isVideoFileExtnameValid } from './custom-validators/videos'
12import { logger } from './logger'
13import { generateVideoImportTmpPath } from './utils'
13 14
14export type YoutubeDLInfo = { 15export type YoutubeDLInfo = {
15 name?: string 16 name?: string
@@ -21,7 +22,6 @@ export type YoutubeDLInfo = {
21 tags?: string[] 22 tags?: string[]
22 thumbnailUrl?: string 23 thumbnailUrl?: string
23 ext?: string 24 ext?: string
24 mergeExt?: string
25 originallyPublishedAt?: Date 25 originallyPublishedAt?: Date
26} 26}
27 27
@@ -51,16 +51,6 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
51 youtubeDL.getInfo(url, args, processOptions, (err, info) => { 51 youtubeDL.getInfo(url, args, processOptions, (err, info) => {
52 if (err) return rej(err) 52 if (err) return rej(err)
53 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) 53 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
54 if (info.format_id?.includes('+')) {
55 // this is a merge format and its extension will be appended
56 if (info.ext === 'mp4') {
57 info.mergeExt = 'mp4'
58 } else if (info.ext === 'webm') {
59 info.mergeExt = 'webm'
60 } else {
61 info.mergeExt = 'mkv'
62 }
63 }
64 54
65 const obj = buildVideoInfo(normalizeObject(info)) 55 const obj = buildVideoInfo(normalizeObject(info))
66 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' 56 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
@@ -133,18 +123,15 @@ function getYoutubeDLVideoFormat () {
133 ].join('/') 123 ].join('/')
134} 124}
135 125
136function downloadYoutubeDLVideo (url: string, extension: string, timeout: number, mergeExtension?: string) { 126function downloadYoutubeDLVideo (url: string, fileExt: string, timeout: number) {
137 let path = generateVideoImportTmpPath(url, extension) 127 // Leave empty the extension, youtube-dl will add it
138 128 const pathWithoutExtension = generateVideoImportTmpPath(url, '')
139 path = mergeExtension
140 ? path.replace(new RegExp(`${extension}$`), mergeExtension)
141 : path
142 129
143 let timer 130 let timer
144 131
145 logger.info('Importing youtubeDL video %s to %s', url, path) 132 logger.info('Importing youtubeDL video %s to %s', url, pathWithoutExtension)
146 133
147 let options = [ '-f', getYoutubeDLVideoFormat(), '-o', path ] 134 let options = [ '-f', getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ]
148 options = wrapWithProxyOptions(options) 135 options = wrapWithProxyOptions(options)
149 136
150 if (process.env.FFMPEG_PATH) { 137 if (process.env.FFMPEG_PATH) {
@@ -156,26 +143,33 @@ function downloadYoutubeDLVideo (url: string, extension: string, timeout: number
156 return new Promise<string>((res, rej) => { 143 return new Promise<string>((res, rej) => {
157 safeGetYoutubeDL() 144 safeGetYoutubeDL()
158 .then(youtubeDL => { 145 .then(youtubeDL => {
159 youtubeDL.exec(url, options, processOptions, err => { 146 youtubeDL.exec(url, options, processOptions, async err => {
160 clearTimeout(timer) 147 clearTimeout(timer)
161 148
162 if (err) { 149 try {
163 remove(path) 150 const path = await guessVideoPathWithExtension(pathWithoutExtension, fileExt)
164 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) 151
152 if (err) {
153 remove(path)
154 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
165 155
156 return rej(err)
157 }
158
159 return res(path)
160 } catch (err) {
166 return rej(err) 161 return rej(err)
167 } 162 }
168
169 return res(path)
170 }) 163 })
171 164
172 timer = setTimeout(() => { 165 timer = setTimeout(() => {
173 const err = new Error('YoutubeDL download timeout.') 166 const err = new Error('YoutubeDL download timeout.')
174 167
175 remove(path) 168 guessVideoPathWithExtension(pathWithoutExtension, fileExt)
169 .then(path => remove(path))
176 .finally(() => rej(err)) 170 .finally(() => rej(err))
177 .catch(err => { 171 .catch(err => {
178 logger.error('Cannot remove %s in youtubeDL timeout.', path, { err }) 172 logger.error('Cannot remove file in youtubeDL timeout.', { err })
179 return rej(err) 173 return rej(err)
180 }) 174 })
181 }, timeout) 175 }, timeout)
@@ -294,6 +288,22 @@ export {
294 288
295// --------------------------------------------------------------------------- 289// ---------------------------------------------------------------------------
296 290
291async function guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
292 if (!isVideoFileExtnameValid(sourceExt)) {
293 throw new Error('Invalid video extension ' + sourceExt)
294 }
295
296 const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
297
298 for (const extension of extensions) {
299 const path = tmpPath + extension
300
301 if (await pathExists(path)) return path
302 }
303
304 throw new Error('Cannot guess path of ' + tmpPath)
305}
306
297function normalizeObject (obj: any) { 307function normalizeObject (obj: any) {
298 const newObj: any = {} 308 const newObj: any = {}
299 309
@@ -324,8 +334,7 @@ function buildVideoInfo (obj: any): YoutubeDLInfo {
324 tags: getTags(obj.tags), 334 tags: getTags(obj.tags),
325 thumbnailUrl: obj.thumbnail || undefined, 335 thumbnailUrl: obj.thumbnail || undefined,
326 originallyPublishedAt: buildOriginallyPublishedAt(obj), 336 originallyPublishedAt: buildOriginallyPublishedAt(obj),
327 ext: obj.ext, 337 ext: obj.ext
328 mergeExt: obj.mergeExt
329 } 338 }
330} 339}
331 340
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index db3112418..c1d1866b0 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -80,7 +80,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
80 } 80 }
81 81
82 return processFile( 82 return processFile(
83 () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT, payload.mergeExt), 83 () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT),
84 videoImport, 84 videoImport,
85 options 85 options
86 ) 86 )
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 2af2a25a6..f9a6250c9 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -80,9 +80,7 @@ export type VideoImportYoutubeDLPayload = {
80 80
81 generateThumbnail: boolean 81 generateThumbnail: boolean
82 generatePreview: boolean 82 generatePreview: boolean
83
84 fileExt?: string 83 fileExt?: string
85 mergeExt?: string
86} 84}
87export type VideoImportTorrentPayload = { 85export type VideoImportTorrentPayload = {
88 type: VideoImportTorrentPayloadType 86 type: VideoImportTorrentPayloadType