Add audio-only option to transcoders and player

This patch adds an audio-only option to PeerTube by means of a new transcoding configuration which creates mp4 files which only contain an audio stream. This new transcoder has a resolution of '0' and is presented in the preferences and in the player resolution menu as 'Audio-only' (localised). When playing such streams the player shows the file thumbnail as background and disables controls autohide.

Audio-only files can be shared and streamed just like any other file. They can be downloaded as well, the resulting file will be an mp4 container with a single audio stream.

This patch is a proof of concept to show the feasibility of 'true' audio-only support. There are better ways of doing this which also enable multiple audio streams for a given video stream (e.g. DASH) but as this would entail a fundamental change in the way PeerTube works it is a bridge too far for a simple proof of concept.

7 files changed, 87 insertions, 10 deletions

diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 70e8aa970..8a00f9835 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -300,6 +300,7 @@ function customConfig (): CustomConfig {
       allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES,
       threads: CONFIG.TRANSCODING.THREADS,
       resolutions: {
+        '0p': CONFIG.TRANSCODING.RESOLUTIONS[ '0p' ],
         '240p': CONFIG.TRANSCODING.RESOLUTIONS[ '240p' ],
         '360p': CONFIG.TRANSCODING.RESOLUTIONS[ '360p' ],
         '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ],
@@ -356,6 +357,7 @@ function convertCustomConfigBody (body: CustomConfig) {
   function keyConverter (k: string) {
     // Transcoding resolutions exception
     if (/^\d{3,4}p$/.exec(k)) return k
+    if (/^0p$/.exec(k)) return k
 
     return snakeCase(k)
   }
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 7a4ac0970..2d9ce2bfa 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -14,6 +14,7 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
 
   // Put in the order we want to proceed jobs
   const resolutions = [
+    VideoResolution.H_NOVIDEO,
     VideoResolution.H_480P,
     VideoResolution.H_360P,
     VideoResolution.H_720P,
@@ -34,10 +35,15 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
 async function getVideoFileSize (path: string) {
   const videoStream = await getVideoStreamFromFile(path)
 
-  return {
-    width: videoStream.width,
-    height: videoStream.height
-  }
+  return videoStream == null 
+    ? {
+        width: 0,
+        height: 0
+      }
+    : {
+        width: videoStream.width,
+        height: videoStream.height
+      }
 }
 
 async function getVideoFileResolution (path: string) {
@@ -52,6 +58,10 @@ async function getVideoFileResolution (path: string) {
 async function getVideoFileFPS (path: string) {
   const videoStream = await getVideoStreamFromFile(path)
 
+  if (videoStream == null) {
+      return 0
+  }
+
   for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
     const valuesText: string = videoStream[key]
     if (!valuesText) continue
@@ -118,7 +128,7 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima
   }
 }
 
-type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio'
+type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'split-audio'
 
 interface BaseTranscodeOptions {
   type: TranscodeOptionsType
@@ -149,7 +159,11 @@ interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
   audioPath: string
 }
 
-type TranscodeOptions = HLSTranscodeOptions | VideoTranscodeOptions | MergeAudioTranscodeOptions | QuickTranscodeOptions
+interface SplitAudioTranscodeOptions extends BaseTranscodeOptions {
+  type: 'split-audio'
+}
+
+type TranscodeOptions = HLSTranscodeOptions | VideoTranscodeOptions | MergeAudioTranscodeOptions | SplitAudioTranscodeOptions | QuickTranscodeOptions
 
 function transcode (options: TranscodeOptions) {
   return new Promise<void>(async (res, rej) => {
@@ -163,6 +177,8 @@ function transcode (options: TranscodeOptions) {
         command = await buildHLSCommand(command, options)
       } else if (options.type === 'merge-audio') {
         command = await buildAudioMergeCommand(command, options)
+      } else if (options.type === 'split-audio') {
+        command = await buildAudioSplitCommand(command, options)
       } else {
         command = await buildx264Command(command, options)
       }
@@ -198,6 +214,7 @@ async function canDoQuickTranscode (path: string): Promise<boolean> {
   const resolution = await getVideoFileResolution(path)
 
   // check video params
+  if (videoStream == null) return false
   if (videoStream[ 'codec_name' ] !== 'h264') return false
   if (videoStream[ 'pix_fmt' ] !== 'yuv420p') return false
   if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
@@ -276,6 +293,12 @@ async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: M
   return command
 }
 
+async function buildAudioSplitCommand (command: ffmpeg.FfmpegCommand, options: SplitAudioTranscodeOptions) {
+  command = await presetAudioSplit(command)
+
+  return command
+}
+
 async function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
   command = await presetCopy(command)
 
@@ -327,7 +350,7 @@ function getVideoStreamFromFile (path: string) {
       if (err) return rej(err)
 
       const videoStream = metadata.streams.find(s => s.codec_type === 'video')
-      if (!videoStream) return rej(new Error('Cannot find video stream of ' + path))
+      //if (!videoStream) return rej(new Error('Cannot find video stream of ' + path))
 
       return res(videoStream)
     })
@@ -482,3 +505,11 @@ async function presetCopy (command: ffmpeg.FfmpegCommand): Promise<ffmpeg.Ffmpeg
     .videoCodec('copy')
     .audioCodec('copy')
 }
+
+
+async function presetAudioSplit (command: ffmpeg.FfmpegCommand): Promise<ffmpeg.FfmpegCommand> {
+  return command
+    .format('mp4')
+    .audioCodec('copy')
+    .noVideo()
+}
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 6d5d55487..c6e478f57 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -168,6 +168,7 @@ const CONFIG = {
     get ALLOW_AUDIO_FILES () { return config.get<boolean>('transcoding.allow_audio_files') },
     get THREADS () { return config.get<number>('transcoding.threads') },
     RESOLUTIONS: {
+      get '0p' () { return config.get<boolean>('transcoding.resolutions.0p') },
       get '240p' () { return config.get<boolean>('transcoding.resolutions.240p') },
       get '360p' () { return config.get<boolean>('transcoding.resolutions.360p') },
       get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') },
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 9243d1742..9dd54837f 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -81,12 +81,52 @@ async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoR
   const videoOutputPath = getVideoFilePath(video, newVideoFile)
   const videoTranscodedPath = join(transcodeDirectory, getVideoFilename(video, newVideoFile))
 
+  const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
+    ? {
+        type: 'split-audio' as 'split-audio',
+        inputPath: videoInputPath,
+        outputPath: videoTranscodedPath,
+        resolution,
+      }
+    : {
+        type: 'video' as 'video',
+        inputPath: videoInputPath,
+        outputPath: videoTranscodedPath,
+        resolution,
+        isPortraitMode: isPortrait
+      }
+
+  await transcode(transcodeOptions)
+
+  return onVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
+}
+
+/**
+ * Extract audio into a separate audio-only mp4.
+ */
+async function splitAudioFile (video: MVideoWithFile) {
+  const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
+  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
+  const extname = '.mp4'
+  const resolution = VideoResolution.H_NOVIDEO
+
+  // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
+  const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile()))
+
+  const newVideoFile = new VideoFileModel({
+    resolution,
+    extname,
+    size: 0,
+    videoId: video.id
+  })
+  const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))
+  const videoTranscodedPath = join(transcodeDirectory, video.getVideoFilename(newVideoFile))
+
   const transcodeOptions = {
-    type: 'video' as 'video',
+    type: 'split-audio' as 'split-audio',
     inputPath: videoInputPath,
     outputPath: videoTranscodedPath,
-    resolution,
-    isPortraitMode: isPortrait
+    resolution
   }
 
   await transcode(transcodeOptions)
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index 1db907f91..d86fa700b 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -37,6 +37,7 @@ const customConfigUpdateValidator = [
   body('transcoding.enabled').isBoolean().withMessage('Should have a valid transcoding enabled boolean'),
   body('transcoding.allowAdditionalExtensions').isBoolean().withMessage('Should have a valid additional extensions boolean'),
   body('transcoding.threads').isInt().withMessage('Should have a valid transcoding threads number'),
+  body('transcoding.resolutions.0p').isBoolean().withMessage('Should have a valid transcoding 0p resolution enabled boolean'),
   body('transcoding.resolutions.240p').isBoolean().withMessage('Should have a valid transcoding 240p resolution enabled boolean'),
   body('transcoding.resolutions.360p').isBoolean().withMessage('Should have a valid transcoding 360p resolution enabled boolean'),
   body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'),
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 3c558d4ea..443fbcb60 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -85,6 +85,7 @@ describe('Test config API validators', function () {
       allowAudioFiles: true,
       threads: 1,
       resolutions: {
+        '0p': false,
         '240p': false,
         '360p': true,
         '480p': true,
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index a494858b3..cf99e5c0a 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -274,6 +274,7 @@ describe('Test config', function () {
         allowAudioFiles: true,
         threads: 1,
         resolutions: {
+          '0p': false,
           '240p': false,
           '360p': true,
           '480p': true,