Fix (again) youtube import

5 files changed, 52 insertions, 52 deletions

diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 9702e219a..96499dffb 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -220,8 +220,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
     videoImportId: videoImport.id,
     generateThumbnail: !thumbnailModel,
     generatePreview: !previewModel,
-    fileExt: `.${youtubeDLInfo.ext || 'mp4'}`,
-    mergeExt: youtubeDLInfo.mergeExt ? `.${youtubeDLInfo.mergeExt}` : ''
+    fileExt: `.${youtubeDLInfo.ext || 'mp4'}`
   }
   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index ad3b7949e..0545e8996 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -1,11 +1,10 @@
+import { remove } from 'fs-extra'
+import { Instance as ParseTorrent } from 'parse-torrent'
+import { join } from 'path'
 import { ResultList } from '../../shared'
+import { CONFIG } from '../initializers/config'
 import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
 import { logger } from './logger'
-import { join } from 'path'
-import { Instance as ParseTorrent } from 'parse-torrent'
-import { remove } from 'fs-extra'
-import { CONFIG } from '../initializers/config'
-import { isVideoFileExtnameValid } from './custom-validators/videos'
 
 function deleteFileAsync (path: string) {
   remove(path)
@@ -31,16 +30,11 @@ function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects:
   } as ResultList<V>
 }
 
-function generateVideoImportTmpPath (target: string | ParseTorrent, extensionArg?: string) {
+function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') {
   const id = typeof target === 'string'
     ? target
     : target.infoHash
 
-  let extension = '.mp4'
-  if (extensionArg && isVideoFileExtnameValid(extensionArg)) {
-    extension = extensionArg
-  }
-
   const hash = sha256(id)
   return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
 }
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 72d457bc1..6b9d8a5f7 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,15 +1,16 @@
-import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
-import { logger } from './logger'
-import { generateVideoImportTmpPath } from './utils'
-import { getEnabledResolutions } from '../lib/video-transcoding'
+import { createWriteStream } from 'fs'
+import { ensureDir, pathExists, remove, writeFile } from 'fs-extra'
 import { join } from 'path'
-import { peertubeTruncate, root } from './core-utils'
-import { ensureDir, remove, writeFile } from 'fs-extra'
 import * as request from 'request'
-import { createWriteStream } from 'fs'
 import { CONFIG } from '@server/initializers/config'
 import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
 import { VideoResolution } from '../../shared/models/videos'
+import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
+import { getEnabledResolutions } from '../lib/video-transcoding'
+import { peertubeTruncate, root } from './core-utils'
+import { isVideoFileExtnameValid } from './custom-validators/videos'
+import { logger } from './logger'
+import { generateVideoImportTmpPath } from './utils'
 
 export type YoutubeDLInfo = {
   name?: string
@@ -21,7 +22,6 @@ export type YoutubeDLInfo = {
   tags?: string[]
   thumbnailUrl?: string
   ext?: string
-  mergeExt?: string
   originallyPublishedAt?: Date
 }
 
@@ -51,16 +51,6 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
         youtubeDL.getInfo(url, args, processOptions, (err, info) => {
           if (err) return rej(err)
           if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
-          if (info.format_id?.includes('+')) {
-            // this is a merge format and its extension will be appended
-            if (info.ext === 'mp4') {
-              info.mergeExt = 'mp4'
-            } else if (info.ext === 'webm') {
-              info.mergeExt = 'webm'
-            } else {
-              info.mergeExt = 'mkv'
-            }
-          }
 
           const obj = buildVideoInfo(normalizeObject(info))
           if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
@@ -133,18 +123,15 @@ function getYoutubeDLVideoFormat () {
   ].join('/')
 }
 
-function downloadYoutubeDLVideo (url: string, extension: string, timeout: number, mergeExtension?: string) {
-  let path = generateVideoImportTmpPath(url, extension)
-
-  path = mergeExtension
-    ? path.replace(new RegExp(`${extension}$`), mergeExtension)
-    : path
+function downloadYoutubeDLVideo (url: string, fileExt: string, timeout: number) {
+  // Leave empty the extension, youtube-dl will add it
+  const pathWithoutExtension = generateVideoImportTmpPath(url, '')
 
   let timer
 
-  logger.info('Importing youtubeDL video %s to %s', url, path)
+  logger.info('Importing youtubeDL video %s to %s', url, pathWithoutExtension)
 
-  let options = [ '-f', getYoutubeDLVideoFormat(), '-o', path ]
+  let options = [ '-f', getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ]
   options = wrapWithProxyOptions(options)
 
   if (process.env.FFMPEG_PATH) {
@@ -156,26 +143,33 @@ function downloadYoutubeDLVideo (url: string, extension: string, timeout: number
   return new Promise<string>((res, rej) => {
     safeGetYoutubeDL()
       .then(youtubeDL => {
-        youtubeDL.exec(url, options, processOptions, err => {
+        youtubeDL.exec(url, options, processOptions, async err => {
           clearTimeout(timer)
 
-          if (err) {
-            remove(path)
-              .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
+          try {
+            const path = await guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+
+            if (err) {
+              remove(path)
+                .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
 
+              return rej(err)
+            }
+
+            return res(path)
+          } catch (err) {
             return rej(err)
           }
-
-          return res(path)
         })
 
         timer = setTimeout(() => {
           const err = new Error('YoutubeDL download timeout.')
 
-          remove(path)
+          guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+            .then(path => remove(path))
             .finally(() => rej(err))
             .catch(err => {
-              logger.error('Cannot remove %s in youtubeDL timeout.', path, { err })
+              logger.error('Cannot remove file in youtubeDL timeout.', { err })
               return rej(err)
             })
         }, timeout)
@@ -294,6 +288,22 @@ export {
 
 // ---------------------------------------------------------------------------
 
+async function guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
+  if (!isVideoFileExtnameValid(sourceExt)) {
+    throw new Error('Invalid video extension ' + sourceExt)
+  }
+
+  const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
+
+  for (const extension of extensions) {
+    const path = tmpPath + extension
+
+    if (await pathExists(path)) return path
+  }
+
+  throw new Error('Cannot guess path of ' + tmpPath)
+}
+
 function normalizeObject (obj: any) {
   const newObj: any = {}
 
@@ -324,8 +334,7 @@ function buildVideoInfo (obj: any): YoutubeDLInfo {
     tags: getTags(obj.tags),
     thumbnailUrl: obj.thumbnail || undefined,
     originallyPublishedAt: buildOriginallyPublishedAt(obj),
-    ext: obj.ext,
-    mergeExt: obj.mergeExt
+    ext: obj.ext
   }
 }
 
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index db3112418..c1d1866b0 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -80,7 +80,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
   }
 
   return processFile(
-    () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT, payload.mergeExt),
+    () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT),
     videoImport,
     options
   )
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 2af2a25a6..f9a6250c9 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -80,9 +80,7 @@ export type VideoImportYoutubeDLPayload = {
 
   generateThumbnail: boolean
   generatePreview: boolean
-
   fileExt?: string
-  mergeExt?: string
 }
 export type VideoImportTorrentPayload = {
   type: VideoImportTorrentPayloadType