server/helpers/ffmpeg-utils.ts

   1 import * as ffmpeg from 'fluent-ffmpeg'
   2 import { dirname, join } from 'path'
   3 import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos'
   4 import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
   5 import { processImage } from './image-utils'
   6 import { logger } from './logger'
   7 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   8 import { readFile, remove, writeFile } from 'fs-extra'
   9 import { CONFIG } from '../initializers/config'
  10 import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
  11
  12 /**
  13  * A toolbox to play with audio
  14  */
  15 namespace audio {
  16   export const get = (videoPath: string) => {
  17     // without position, ffprobe considers the last input only
  18     // we make it consider the first input only
  19     // if you pass a file path to pos, then ffprobe acts on that file directly
  20     return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
  21
  22       function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
  23         if (err) return rej(err)
  24
  25         if ('streams' in data) {
  26           const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
  27           if (audioStream) {
  28             return res({
  29               absolutePath: data.format.filename,
  30               audioStream
  31             })
  32           }
  33         }
  34
  35         return res({ absolutePath: data.format.filename })
  36       }
  37
  38       return ffmpeg.ffprobe(videoPath, parseFfprobe)
  39     })
  40   }
  41
  42   export namespace bitrate {
  43     const baseKbitrate = 384
  44
  45     const toBits = (kbits: number) => kbits * 8000
  46
  47     export const aac = (bitrate: number): number => {
  48       switch (true) {
  49         case bitrate > toBits(baseKbitrate):
  50           return baseKbitrate
  51
  52         default:
  53           return -1 // we interpret it as a signal to copy the audio stream as is
  54       }
  55     }
  56
  57     export const mp3 = (bitrate: number): number => {
  58       /*
  59       a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
  60       That's why, when using aac, we can go to lower kbit/sec. The equivalences
  61       made here are not made to be accurate, especially with good mp3 encoders.
  62       */
  63       switch (true) {
  64         case bitrate <= toBits(192):
  65           return 128
  66
  67         case bitrate <= toBits(384):
  68           return 256
  69
  70         default:
  71           return baseKbitrate
  72       }
  73     }
  74   }
  75 }
  76
  77 function computeResolutionsToTranscode (videoFileHeight: number) {
  78   const resolutionsEnabled: number[] = []
  79   const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
  80
  81   // Put in the order we want to proceed jobs
  82   const resolutions = [
  83     VideoResolution.H_NOVIDEO,
  84     VideoResolution.H_480P,
  85     VideoResolution.H_360P,
  86     VideoResolution.H_720P,
  87     VideoResolution.H_240P,
  88     VideoResolution.H_1080P,
  89     VideoResolution.H_4K
  90   ]
  91
  92   for (const resolution of resolutions) {
  93     if (configResolutions[resolution + 'p'] === true && videoFileHeight > resolution) {
  94       resolutionsEnabled.push(resolution)
  95     }
  96   }
  97
  98   return resolutionsEnabled
  99 }
 100
 101 async function getVideoStreamSize (path: string) {
 102   const videoStream = await getVideoStreamFromFile(path)
 103
 104   return videoStream === null
 105     ? { width: 0, height: 0 }
 106     : { width: videoStream.width, height: videoStream.height }
 107 }
 108
 109 async function getVideoStreamCodec (path: string) {
 110   const videoStream = await getVideoStreamFromFile(path)
 111
 112   if (!videoStream) return ''
 113
 114   const videoCodec = videoStream.codec_tag_string
 115
 116   const baseProfileMatrix = {
 117     High: '6400',
 118     Main: '4D40',
 119     Baseline: '42E0'
 120   }
 121
 122   let baseProfile = baseProfileMatrix[videoStream.profile]
 123   if (!baseProfile) {
 124     logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
 125     baseProfile = baseProfileMatrix['High'] // Fallback
 126   }
 127
 128   const level = videoStream.level.toString(16)
 129
 130   return `${videoCodec}.${baseProfile}${level}`
 131 }
 132
 133 async function getAudioStreamCodec (path: string) {
 134   const { audioStream } = await audio.get(path)
 135
 136   if (!audioStream) return ''
 137
 138   const audioCodec = audioStream.codec_name
 139   if (audioCodec === 'aac') return 'mp4a.40.2'
 140
 141   logger.warn('Cannot get audio codec of %s.', path, { audioStream })
 142
 143   return 'mp4a.40.2' // Fallback
 144 }
 145
 146 async function getVideoFileResolution (path: string) {
 147   const size = await getVideoStreamSize(path)
 148
 149   return {
 150     videoFileResolution: Math.min(size.height, size.width),
 151     isPortraitMode: size.height > size.width
 152   }
 153 }
 154
 155 async function getVideoFileFPS (path: string) {
 156   const videoStream = await getVideoStreamFromFile(path)
 157   if (videoStream === null) return 0
 158
 159   for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
 160     const valuesText: string = videoStream[key]
 161     if (!valuesText) continue
 162
 163     const [ frames, seconds ] = valuesText.split('/')
 164     if (!frames || !seconds) continue
 165
 166     const result = parseInt(frames, 10) / parseInt(seconds, 10)
 167     if (result > 0) return Math.round(result)
 168   }
 169
 170   return 0
 171 }
 172
 173 async function getMetadataFromFile <T> (path: string, cb = metadata => metadata) {
 174   return new Promise<T>((res, rej) => {
 175     ffmpeg.ffprobe(path, (err, metadata) => {
 176       if (err) return rej(err)
 177
 178       return res(cb(new VideoFileMetadata(metadata)))
 179     })
 180   })
 181 }
 182
 183 async function getVideoFileBitrate (path: string) {
 184   return getMetadataFromFile<number>(path, metadata => metadata.format.bit_rate)
 185 }
 186
 187 function getDurationFromVideoFile (path: string) {
 188   return getMetadataFromFile<number>(path, metadata => Math.floor(metadata.format.duration))
 189 }
 190
 191 function getVideoStreamFromFile (path: string) {
 192   return getMetadataFromFile<any>(path, metadata => metadata.streams.find(s => s.codec_type === 'video') || null)
 193 }
 194
 195 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
 196   const pendingImageName = 'pending-' + imageName
 197
 198   const options = {
 199     filename: pendingImageName,
 200     count: 1,
 201     folder
 202   }
 203
 204   const pendingImagePath = join(folder, pendingImageName)
 205
 206   try {
 207     await new Promise<string>((res, rej) => {
 208       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
 209         .on('error', rej)
 210         .on('end', () => res(imageName))
 211         .thumbnail(options)
 212     })
 213
 214     const destination = join(folder, imageName)
 215     await processImage(pendingImagePath, destination, size)
 216   } catch (err) {
 217     logger.error('Cannot generate image from video %s.', fromPath, { err })
 218
 219     try {
 220       await remove(pendingImagePath)
 221     } catch (err) {
 222       logger.debug('Cannot remove pending image path after generation error.', { err })
 223     }
 224   }
 225 }
 226
 227 type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 228
 229 interface BaseTranscodeOptions {
 230   type: TranscodeOptionsType
 231   inputPath: string
 232   outputPath: string
 233   resolution: VideoResolution
 234   isPortraitMode?: boolean
 235 }
 236
 237 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 238   type: 'hls'
 239   copyCodecs: boolean
 240   hlsPlaylist: {
 241     videoFilename: string
 242   }
 243 }
 244
 245 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 246   type: 'quick-transcode'
 247 }
 248
 249 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 250   type: 'video'
 251 }
 252
 253 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 254   type: 'merge-audio'
 255   audioPath: string
 256 }
 257
 258 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 259   type: 'only-audio'
 260 }
 261
 262 type TranscodeOptions =
 263   HLSTranscodeOptions
 264   | VideoTranscodeOptions
 265   | MergeAudioTranscodeOptions
 266   | OnlyAudioTranscodeOptions
 267   | QuickTranscodeOptions
 268
 269 function transcode (options: TranscodeOptions) {
 270   return new Promise<void>(async (res, rej) => {
 271     try {
 272       let command = ffmpeg(options.inputPath, { niceness: FFMPEG_NICE.TRANSCODING })
 273         .output(options.outputPath)
 274
 275       if (options.type === 'quick-transcode') {
 276         command = buildQuickTranscodeCommand(command)
 277       } else if (options.type === 'hls') {
 278         command = await buildHLSCommand(command, options)
 279       } else if (options.type === 'merge-audio') {
 280         command = await buildAudioMergeCommand(command, options)
 281       } else if (options.type === 'only-audio') {
 282         command = buildOnlyAudioCommand(command, options)
 283       } else {
 284         command = await buildx264Command(command, options)
 285       }
 286
 287       if (CONFIG.TRANSCODING.THREADS > 0) {
 288         // if we don't set any threads ffmpeg will chose automatically
 289         command = command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
 290       }
 291
 292       command
 293         .on('error', (err, stdout, stderr) => {
 294           logger.error('Error in transcoding job.', { stdout, stderr })
 295           return rej(err)
 296         })
 297         .on('end', () => {
 298           return fixHLSPlaylistIfNeeded(options)
 299             .then(() => res())
 300             .catch(err => rej(err))
 301         })
 302         .run()
 303     } catch (err) {
 304       return rej(err)
 305     }
 306   })
 307 }
 308
 309 async function canDoQuickTranscode (path: string): Promise<boolean> {
 310   // NOTE: This could be optimized by running ffprobe only once (but it runs fast anyway)
 311   const videoStream = await getVideoStreamFromFile(path)
 312   const parsedAudio = await audio.get(path)
 313   const fps = await getVideoFileFPS(path)
 314   const bitRate = await getVideoFileBitrate(path)
 315   const resolution = await getVideoFileResolution(path)
 316
 317   // check video params
 318   if (videoStream == null) return false
 319   if (videoStream['codec_name'] !== 'h264') return false
 320   if (videoStream['pix_fmt'] !== 'yuv420p') return false
 321   if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
 322   if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false
 323
 324   // check audio params (if audio stream exists)
 325   if (parsedAudio.audioStream) {
 326     if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
 327
 328     const maxAudioBitrate = audio.bitrate['aac'](parsedAudio.audioStream['bit_rate'])
 329     if (maxAudioBitrate !== -1 && parsedAudio.audioStream['bit_rate'] > maxAudioBitrate) return false
 330   }
 331
 332   return true
 333 }
 334
 335 function getClosestFramerateStandard (fps: number, type: 'HD_STANDARD' | 'STANDARD'): number {
 336   return VIDEO_TRANSCODING_FPS[type].slice(0)
 337                                     .sort((a, b) => fps % a - fps % b)[0]
 338 }
 339
 340 // ---------------------------------------------------------------------------
 341
 342 export {
 343   getVideoStreamCodec,
 344   getAudioStreamCodec,
 345   getVideoStreamSize,
 346   getVideoFileResolution,
 347   getMetadataFromFile,
 348   getDurationFromVideoFile,
 349   generateImageFromVideoFile,
 350   TranscodeOptions,
 351   TranscodeOptionsType,
 352   transcode,
 353   getVideoFileFPS,
 354   computeResolutionsToTranscode,
 355   audio,
 356   getVideoFileBitrate,
 357   canDoQuickTranscode
 358 }
 359
 360 // ---------------------------------------------------------------------------
 361
 362 async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 363   let fps = await getVideoFileFPS(options.inputPath)
 364   if (
 365     // On small/medium resolutions, limit FPS
 366     options.resolution !== undefined &&
 367     options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
 368     fps > VIDEO_TRANSCODING_FPS.AVERAGE
 369   ) {
 370     // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
 371     fps = getClosestFramerateStandard(fps, 'STANDARD')
 372   }
 373
 374   command = await presetH264(command, options.inputPath, options.resolution, fps)
 375
 376   if (options.resolution !== undefined) {
 377     // '?x720' or '720x?' for example
 378     const size = options.isPortraitMode === true ? `${options.resolution}x?` : `?x${options.resolution}`
 379     command = command.size(size)
 380   }
 381
 382   if (fps) {
 383     // Hard FPS limits
 384     if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
 385     else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
 386
 387     command = command.withFPS(fps)
 388   }
 389
 390   return command
 391 }
 392
 393 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 394   command = command.loop(undefined)
 395
 396   command = await presetH264VeryFast(command, options.audioPath, options.resolution)
 397
 398   command = command.input(options.audioPath)
 399                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 400                    .outputOption('-tune stillimage')
 401                    .outputOption('-shortest')
 402
 403   return command
 404 }
 405
 406 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) {
 407   command = presetOnlyAudio(command)
 408
 409   return command
 410 }
 411
 412 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 413   command = presetCopy(command)
 414
 415   command = command.outputOption('-map_metadata -1') // strip all metadata
 416                    .outputOption('-movflags faststart')
 417
 418   return command
 419 }
 420
 421 async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 422   const videoPath = getHLSVideoPath(options)
 423
 424   if (options.copyCodecs) command = presetCopy(command)
 425   else command = await buildx264Command(command, options)
 426
 427   command = command.outputOption('-hls_time 4')
 428                    .outputOption('-hls_list_size 0')
 429                    .outputOption('-hls_playlist_type vod')
 430                    .outputOption('-hls_segment_filename ' + videoPath)
 431                    .outputOption('-hls_segment_type fmp4')
 432                    .outputOption('-f hls')
 433                    .outputOption('-hls_flags single_file')
 434
 435   return command
 436 }
 437
 438 function getHLSVideoPath (options: HLSTranscodeOptions) {
 439   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 440 }
 441
 442 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 443   if (options.type !== 'hls') return
 444
 445   const fileContent = await readFile(options.outputPath)
 446
 447   const videoFileName = options.hlsPlaylist.videoFilename
 448   const videoFilePath = getHLSVideoPath(options)
 449
 450   // Fix wrong mapping with some ffmpeg versions
 451   const newContent = fileContent.toString()
 452                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 453
 454   await writeFile(options.outputPath, newContent)
 455 }
 456
 457 /**
 458  * A slightly customised version of the 'veryfast' x264 preset
 459  *
 460  * The veryfast preset is right in the sweet spot of performance
 461  * and quality. Superfast and ultrafast will give you better
 462  * performance, but then quality is noticeably worse.
 463  */
 464 async function presetH264VeryFast (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 465   let localCommand = await presetH264(command, input, resolution, fps)
 466
 467   localCommand = localCommand.outputOption('-preset:v veryfast')
 468
 469   /*
 470   MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html
 471   Our target situation is closer to a livestream than a stream,
 472   since we want to reduce as much a possible the encoding burden,
 473   although not to the point of a livestream where there is a hard
 474   constraint on the frames per second to be encoded.
 475   */
 476
 477   return localCommand
 478 }
 479
 480 /**
 481  * Standard profile, with variable bitrate audio and faststart.
 482  *
 483  * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel
 484  * See https://trac.ffmpeg.org/wiki/Encode/AAC#fdk_vbr
 485  */
 486 async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 487   let localCommand = command
 488     .format('mp4')
 489     .videoCodec('libx264')
 490     .outputOption('-level 3.1') // 3.1 is the minimal resource allocation for our highest supported resolution
 491     .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 492     .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 493     .outputOption('-pix_fmt yuv420p') // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 494     .outputOption('-map_metadata -1') // strip all metadata
 495     .outputOption('-movflags faststart')
 496
 497   const parsedAudio = await audio.get(input)
 498
 499   if (!parsedAudio.audioStream) {
 500     localCommand = localCommand.noAudio()
 501   } else if ((await checkFFmpegEncoders()).get('libfdk_aac')) { // we favor VBR, if a good AAC encoder is available
 502     localCommand = localCommand
 503       .audioCodec('libfdk_aac')
 504       .audioQuality(5)
 505   } else {
 506     // we try to reduce the ceiling bitrate by making rough matches of bitrates
 507     // of course this is far from perfect, but it might save some space in the end
 508     localCommand = localCommand.audioCodec('aac')
 509
 510     const audioCodecName = parsedAudio.audioStream['codec_name']
 511
 512     if (audio.bitrate[audioCodecName]) {
 513       const bitrate = audio.bitrate[audioCodecName](parsedAudio.audioStream['bit_rate'])
 514       if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate)
 515     }
 516   }
 517
 518   if (fps) {
 519     // Constrained Encoding (VBV)
 520     // https://slhck.info/video/2017/03/01/rate-control.html
 521     // https://trac.ffmpeg.org/wiki/Limiting%20the%20output%20bitrate
 522     const targetBitrate = getTargetBitrate(resolution, fps, VIDEO_TRANSCODING_FPS)
 523     localCommand = localCommand.outputOptions([ `-maxrate ${targetBitrate}`, `-bufsize ${targetBitrate * 2}` ])
 524
 525     // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 526     // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 527     // https://superuser.com/a/908325
 528     localCommand = localCommand.outputOption(`-g ${fps * 2}`)
 529   }
 530
 531   return localCommand
 532 }
 533
 534 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 535   return command
 536     .format('mp4')
 537     .videoCodec('copy')
 538     .audioCodec('copy')
 539 }
 540
 541 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 542   return command
 543     .format('mp4')
 544     .audioCodec('copy')
 545     .noVideo()
 546 }