Support audio files import

6 files changed, 73 insertions, 46 deletions

diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index ed223cbc9..da0832258 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -174,7 +174,10 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
     videoImportId: videoImport.id,
     thumbnailUrl: youtubeDLInfo.thumbnailUrl,
     downloadThumbnail: !thumbnailModel,
-    downloadPreview: !previewModel
+    downloadPreview: !previewModel,
+    fileExt: youtubeDLInfo.fileExt
+      ? `.${youtubeDLInfo.fileExt}`
+      : '.mp4'
   }
   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 9b19c394d..04d775cbf 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -1,7 +1,7 @@
 import * as express from 'express'
 import { extname } from 'path'
 import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../shared'
-import { getVideoFileFPS, getVideoFileResolution, getMetadataFromFile } from '../../../helpers/ffmpeg-utils'
+import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
 import { logger } from '../../../helpers/logger'
 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
 import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
@@ -32,13 +32,13 @@ import {
   paginationValidator,
   setDefaultPagination,
   setDefaultSort,
+  videoFileMetadataGetValidator,
   videosAddValidator,
   videosCustomGetValidator,
   videosGetValidator,
   videosRemoveValidator,
   videosSortValidator,
-  videosUpdateValidator,
-  videoFileMetadataGetValidator
+  videosUpdateValidator
 } from '../../../middlewares'
 import { TagModel } from '../../../models/video/tag'
 import { VideoModel } from '../../../models/video/video'
@@ -62,12 +62,12 @@ import { CONFIG } from '../../../initializers/config'
 import { sequelizeTypescript } from '../../../initializers/database'
 import { createVideoMiniatureFromExisting, generateVideoMiniature } from '../../../lib/thumbnail'
 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
-import { VideoTranscodingPayload } from '../../../lib/job-queue/handlers/video-transcoding'
 import { Hooks } from '../../../lib/plugins/hooks'
 import { MVideoDetails, MVideoFullLight } from '@server/typings/models'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getVideoFilePath } from '@server/lib/video-paths'
 import toInt from 'validator/lib/toInt'
+import { addOptimizeOrMergeAudioJob } from '@server/lib/videos'
 
 const auditLogger = auditLoggerFactory('videos')
 const videosRouter = express.Router()
@@ -296,25 +296,7 @@ async function addVideo (req: express.Request, res: express.Response) {
   Notifier.Instance.notifyOnNewVideoIfNeeded(videoCreated)
 
   if (video.state === VideoState.TO_TRANSCODE) {
-    // Put uuid because we don't have id auto incremented for now
-    let dataInput: VideoTranscodingPayload
-
-    if (videoFile.isAudio()) {
-      dataInput = {
-        type: 'merge-audio' as 'merge-audio',
-        resolution: DEFAULT_AUDIO_RESOLUTION,
-        videoUUID: videoCreated.uuid,
-        isNewVideo: true
-      }
-    } else {
-      dataInput = {
-        type: 'optimize' as 'optimize',
-        videoUUID: videoCreated.uuid,
-        isNewVideo: true
-      }
-    }
-
-    await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
+    await addOptimizeOrMergeAudioJob(videoCreated, videoFile)
   }
 
   Hooks.runAction('action:api.video.uploaded', { video: videoCreated })
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 7a4c781cc..11c118292 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -7,6 +7,7 @@ import { Instance as ParseTorrent } from 'parse-torrent'
 import { remove } from 'fs-extra'
 import * as memoizee from 'memoizee'
 import { CONFIG } from '../initializers/config'
+import { isVideoFileExtnameValid } from './custom-validators/videos'
 
 function deleteFileAsync (path: string) {
   remove(path)
@@ -42,11 +43,18 @@ const getServerActor = memoizee(async function () {
   return actor
 }, { promise: true })
 
-function generateVideoImportTmpPath (target: string | ParseTorrent) {
-  const id = typeof target === 'string' ? target : target.infoHash
+function generateVideoImportTmpPath (target: string | ParseTorrent, extensionArg?: string) {
+  const id = typeof target === 'string'
+    ? target
+    : target.infoHash
+
+  let extension = '.mp4'
+  if (extensionArg && isVideoFileExtnameValid(extensionArg)) {
+    extension = extensionArg
+  }
 
   const hash = sha256(id)
-  return join(CONFIG.STORAGE.TMP_DIR, hash + '-import.mp4')
+  return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
 }
 
 function getSecureTorrentName (originalName: string) {
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 26dbe6543..07c85797a 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -16,6 +16,7 @@ export type YoutubeDLInfo = {
   nsfw?: boolean
   tags?: string[]
   thumbnailUrl?: string
+  fileExt?: string
   originallyPublishedAt?: Date
 }
 
@@ -44,11 +45,11 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
   })
 }
 
-function downloadYoutubeDLVideo (url: string, timeout: number) {
-  const path = generateVideoImportTmpPath(url)
+function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
+  const path = generateVideoImportTmpPath(url, extension)
   let timer
 
-  logger.info('Importing youtubeDL video %s', url)
+  logger.info('Importing youtubeDL video %s to %s', url, path)
 
   let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
   options = wrapWithProxyOptions(options)
@@ -219,7 +220,8 @@ function buildVideoInfo (obj: any) {
     nsfw: isNSFW(obj),
     tags: getTags(obj.tags),
     thumbnailUrl: obj.thumbnail || undefined,
-    originallyPublishedAt: buildOriginallyPublishedAt(obj)
+    originallyPublishedAt: buildOriginallyPublishedAt(obj),
+    fileExt: obj.ext
   }
 }
 
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 09f225cec..d8052da72 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -8,7 +8,6 @@ import { extname } from 'path'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
 import { VideoState } from '../../../../shared'
-import { JobQueue } from '../index'
 import { federateVideoIfNeeded } from '../../activitypub'
 import { VideoModel } from '../../../models/video/video'
 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
@@ -22,6 +21,7 @@ import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
 import { MThumbnail } from '../../../typings/models/video/thumbnail'
 import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
 import { getVideoFilePath } from '@server/lib/video-paths'
+import { addOptimizeOrMergeAudioJob } from '@server/lib/videos'
 
 type VideoImportYoutubeDLPayload = {
   type: 'youtube-dl'
@@ -30,6 +30,8 @@ type VideoImportYoutubeDLPayload = {
   thumbnailUrl: string
   downloadThumbnail: boolean
   downloadPreview: boolean
+
+  fileExt?: string
 }
 
 type VideoImportTorrentPayload = {
@@ -90,7 +92,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
     generatePreview: false
   }
 
-  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, VIDEO_IMPORT_TIMEOUT), videoImport, options)
+  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), videoImport, options)
 }
 
 async function getVideoImportOrDie (videoImportId: number) {
@@ -154,16 +156,28 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     // Process thumbnail
     let thumbnailModel: MThumbnail
     if (options.downloadThumbnail && options.thumbnailUrl) {
-      thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.MINIATURE)
-    } else if (options.generateThumbnail || options.downloadThumbnail) {
+      try {
+        thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.MINIATURE)
+      } catch (err) {
+        logger.warn('Cannot generate video thumbnail %s for %s.', options.thumbnailUrl, videoImportWithFiles.Video.url, { err })
+      }
+    }
+
+    if (!thumbnailModel && (options.generateThumbnail || options.downloadThumbnail)) {
       thumbnailModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.MINIATURE)
     }
 
     // Process preview
     let previewModel: MThumbnail
     if (options.downloadPreview && options.thumbnailUrl) {
-      previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.PREVIEW)
-    } else if (options.generatePreview || options.downloadPreview) {
+      try {
+        previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.PREVIEW)
+      } catch (err) {
+        logger.warn('Cannot generate video preview %s for %s.', options.thumbnailUrl, videoImportWithFiles.Video.url, { err })
+      }
+    }
+
+    if (!previewModel && (options.generatePreview || options.downloadPreview)) {
       previewModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.PREVIEW)
     }
 
@@ -214,14 +228,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
 
     // Create transcoding jobs?
     if (video.state === VideoState.TO_TRANSCODE) {
-      // Put uuid because we don't have id auto incremented for now
-      const dataInput = {
-        type: 'optimize' as 'optimize',
-        videoUUID: videoImportUpdated.Video.uuid,
-        isNewVideo: true
-      }
-
-      await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
+      await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile)
     }
 
   } catch (err) {
diff --git a/server/lib/videos.ts b/server/lib/videos.ts
index 22e9afbf9..96bdd42e9 100644
--- a/server/lib/videos.ts
+++ b/server/lib/videos.ts
@@ -1,4 +1,7 @@
-import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typings/models'
+import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/typings/models'
+import { VideoTranscodingPayload } from '@server/lib/job-queue/handlers/video-transcoding'
+import { DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants'
+import { JobQueue } from '@server/lib/job-queue'
 
 function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
   return isStreamingPlaylist(videoOrPlaylist)
@@ -6,6 +9,28 @@ function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
     : videoOrPlaylist
 }
 
+function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile) {
+  let dataInput: VideoTranscodingPayload
+
+  if (videoFile.isAudio()) {
+    dataInput = {
+      type: 'merge-audio' as 'merge-audio',
+      resolution: DEFAULT_AUDIO_RESOLUTION,
+      videoUUID: video.uuid,
+      isNewVideo: true
+    }
+  } else {
+    dataInput = {
+      type: 'optimize' as 'optimize',
+      videoUUID: video.uuid,
+      isNewVideo: true
+    }
+  }
+
+  return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
+}
+
 export {
+  addOptimizeOrMergeAudioJob,
   extractVideo
 }