server/helpers/ffmpeg-utils.ts

   1 import * as ffmpeg from 'fluent-ffmpeg'
   2 import { dirname, join } from 'path'
   3 import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos'
   4 import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
   5 import { processImage } from './image-utils'
   6 import { logger } from './logger'
   7 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   8 import { readFile, remove, writeFile } from 'fs-extra'
   9 import { CONFIG } from '../initializers/config'
  10 import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
  11
  12 /**
  13  * A toolbox to play with audio
  14  */
  15 namespace audio {
  16   export const get = (videoPath: string) => {
  17     // without position, ffprobe considers the last input only
  18     // we make it consider the first input only
  19     // if you pass a file path to pos, then ffprobe acts on that file directly
  20     return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
  21
  22       function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
  23         if (err) return rej(err)
  24
  25         if ('streams' in data) {
  26           const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
  27           if (audioStream) {
  28             return res({
  29               absolutePath: data.format.filename,
  30               audioStream
  31             })
  32           }
  33         }
  34
  35         return res({ absolutePath: data.format.filename })
  36       }
  37
  38       return ffmpeg.ffprobe(videoPath, parseFfprobe)
  39     })
  40   }
  41
  42   export namespace bitrate {
  43     const baseKbitrate = 384
  44
  45     const toBits = (kbits: number) => kbits * 8000
  46
  47     export const aac = (bitrate: number): number => {
  48       switch (true) {
  49         case bitrate > toBits(baseKbitrate):
  50           return baseKbitrate
  51
  52         default:
  53           return -1 // we interpret it as a signal to copy the audio stream as is
  54       }
  55     }
  56
  57     export const mp3 = (bitrate: number): number => {
  58       /*
  59       a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
  60       That's why, when using aac, we can go to lower kbit/sec. The equivalences
  61       made here are not made to be accurate, especially with good mp3 encoders.
  62       */
  63       switch (true) {
  64         case bitrate <= toBits(192):
  65           return 128
  66
  67         case bitrate <= toBits(384):
  68           return 256
  69
  70         default:
  71           return baseKbitrate
  72       }
  73     }
  74   }
  75 }
  76
  77 function computeResolutionsToTranscode (videoFileHeight: number) {
  78   const resolutionsEnabled: number[] = []
  79   const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
  80
  81   // Put in the order we want to proceed jobs
  82   const resolutions = [
  83     VideoResolution.H_NOVIDEO,
  84     VideoResolution.H_480P,
  85     VideoResolution.H_360P,
  86     VideoResolution.H_720P,
  87     VideoResolution.H_240P,
  88     VideoResolution.H_1080P,
  89     VideoResolution.H_4K
  90   ]
  91
  92   for (const resolution of resolutions) {
  93     if (configResolutions[resolution + 'p'] === true && videoFileHeight > resolution) {
  94       resolutionsEnabled.push(resolution)
  95     }
  96   }
  97
  98   return resolutionsEnabled
  99 }
 100
 101 async function getVideoStreamSize (path: string) {
 102   const videoStream = await getVideoStreamFromFile(path)
 103
 104   return videoStream === null
 105     ? { width: 0, height: 0 }
 106     : { width: videoStream.width, height: videoStream.height }
 107 }
 108
 109 async function getVideoStreamCodec (path: string) {
 110   const videoStream = await getVideoStreamFromFile(path)
 111
 112   if (!videoStream) return ''
 113
 114   const videoCodec = videoStream.codec_tag_string
 115
 116   const baseProfileMatrix = {
 117     High: '6400',
 118     Main: '4D40',
 119     Baseline: '42E0'
 120   }
 121
 122   let baseProfile = baseProfileMatrix[videoStream.profile]
 123   if (!baseProfile) {
 124     logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
 125     baseProfile = baseProfileMatrix['High'] // Fallback
 126   }
 127
 128   let level = videoStream.level.toString(16)
 129   if (level.length === 1) level = `0${level}`
 130
 131   return `${videoCodec}.${baseProfile}${level}`
 132 }
 133
 134 async function getAudioStreamCodec (path: string) {
 135   const { audioStream } = await audio.get(path)
 136
 137   if (!audioStream) return ''
 138
 139   const audioCodec = audioStream.codec_name
 140   if (audioCodec === 'aac') return 'mp4a.40.2'
 141
 142   logger.warn('Cannot get audio codec of %s.', path, { audioStream })
 143
 144   return 'mp4a.40.2' // Fallback
 145 }
 146
 147 async function getVideoFileResolution (path: string) {
 148   const size = await getVideoStreamSize(path)
 149
 150   return {
 151     videoFileResolution: Math.min(size.height, size.width),
 152     isPortraitMode: size.height > size.width
 153   }
 154 }
 155
 156 async function getVideoFileFPS (path: string) {
 157   const videoStream = await getVideoStreamFromFile(path)
 158   if (videoStream === null) return 0
 159
 160   for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
 161     const valuesText: string = videoStream[key]
 162     if (!valuesText) continue
 163
 164     const [ frames, seconds ] = valuesText.split('/')
 165     if (!frames || !seconds) continue
 166
 167     const result = parseInt(frames, 10) / parseInt(seconds, 10)
 168     if (result > 0) return Math.round(result)
 169   }
 170
 171   return 0
 172 }
 173
 174 async function getMetadataFromFile <T> (path: string, cb = metadata => metadata) {
 175   return new Promise<T>((res, rej) => {
 176     ffmpeg.ffprobe(path, (err, metadata) => {
 177       if (err) return rej(err)
 178
 179       return res(cb(new VideoFileMetadata(metadata)))
 180     })
 181   })
 182 }
 183
 184 async function getVideoFileBitrate (path: string) {
 185   return getMetadataFromFile<number>(path, metadata => metadata.format.bit_rate)
 186 }
 187
 188 function getDurationFromVideoFile (path: string) {
 189   return getMetadataFromFile<number>(path, metadata => Math.floor(metadata.format.duration))
 190 }
 191
 192 function getVideoStreamFromFile (path: string) {
 193   return getMetadataFromFile<any>(path, metadata => metadata.streams.find(s => s.codec_type === 'video') || null)
 194 }
 195
 196 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
 197   const pendingImageName = 'pending-' + imageName
 198
 199   const options = {
 200     filename: pendingImageName,
 201     count: 1,
 202     folder
 203   }
 204
 205   const pendingImagePath = join(folder, pendingImageName)
 206
 207   try {
 208     await new Promise<string>((res, rej) => {
 209       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
 210         .on('error', rej)
 211         .on('end', () => res(imageName))
 212         .thumbnail(options)
 213     })
 214
 215     const destination = join(folder, imageName)
 216     await processImage(pendingImagePath, destination, size)
 217   } catch (err) {
 218     logger.error('Cannot generate image from video %s.', fromPath, { err })
 219
 220     try {
 221       await remove(pendingImagePath)
 222     } catch (err) {
 223       logger.debug('Cannot remove pending image path after generation error.', { err })
 224     }
 225   }
 226 }
 227
 228 type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 229
 230 interface BaseTranscodeOptions {
 231   type: TranscodeOptionsType
 232   inputPath: string
 233   outputPath: string
 234   resolution: VideoResolution
 235   isPortraitMode?: boolean
 236 }
 237
 238 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 239   type: 'hls'
 240   copyCodecs: boolean
 241   hlsPlaylist: {
 242     videoFilename: string
 243   }
 244 }
 245
 246 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 247   type: 'quick-transcode'
 248 }
 249
 250 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 251   type: 'video'
 252 }
 253
 254 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 255   type: 'merge-audio'
 256   audioPath: string
 257 }
 258
 259 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 260   type: 'only-audio'
 261 }
 262
 263 type TranscodeOptions =
 264   HLSTranscodeOptions
 265   | VideoTranscodeOptions
 266   | MergeAudioTranscodeOptions
 267   | OnlyAudioTranscodeOptions
 268   | QuickTranscodeOptions
 269
 270 function transcode (options: TranscodeOptions) {
 271   return new Promise<void>(async (res, rej) => {
 272     try {
 273       let command = ffmpeg(options.inputPath, { niceness: FFMPEG_NICE.TRANSCODING })
 274         .output(options.outputPath)
 275
 276       if (options.type === 'quick-transcode') {
 277         command = buildQuickTranscodeCommand(command)
 278       } else if (options.type === 'hls') {
 279         command = await buildHLSCommand(command, options)
 280       } else if (options.type === 'merge-audio') {
 281         command = await buildAudioMergeCommand(command, options)
 282       } else if (options.type === 'only-audio') {
 283         command = buildOnlyAudioCommand(command, options)
 284       } else {
 285         command = await buildx264Command(command, options)
 286       }
 287
 288       if (CONFIG.TRANSCODING.THREADS > 0) {
 289         // if we don't set any threads ffmpeg will chose automatically
 290         command = command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
 291       }
 292
 293       command
 294         .on('error', (err, stdout, stderr) => {
 295           logger.error('Error in transcoding job.', { stdout, stderr })
 296           return rej(err)
 297         })
 298         .on('end', () => {
 299           return fixHLSPlaylistIfNeeded(options)
 300             .then(() => res())
 301             .catch(err => rej(err))
 302         })
 303         .run()
 304     } catch (err) {
 305       return rej(err)
 306     }
 307   })
 308 }
 309
 310 async function canDoQuickTranscode (path: string): Promise<boolean> {
 311   // NOTE: This could be optimized by running ffprobe only once (but it runs fast anyway)
 312   const videoStream = await getVideoStreamFromFile(path)
 313   const parsedAudio = await audio.get(path)
 314   const fps = await getVideoFileFPS(path)
 315   const bitRate = await getVideoFileBitrate(path)
 316   const resolution = await getVideoFileResolution(path)
 317
 318   // check video params
 319   if (videoStream == null) return false
 320   if (videoStream['codec_name'] !== 'h264') return false
 321   if (videoStream['pix_fmt'] !== 'yuv420p') return false
 322   if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
 323   if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false
 324
 325   // check audio params (if audio stream exists)
 326   if (parsedAudio.audioStream) {
 327     if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
 328
 329     const maxAudioBitrate = audio.bitrate['aac'](parsedAudio.audioStream['bit_rate'])
 330     if (maxAudioBitrate !== -1 && parsedAudio.audioStream['bit_rate'] > maxAudioBitrate) return false
 331   }
 332
 333   return true
 334 }
 335
 336 function getClosestFramerateStandard (fps: number, type: 'HD_STANDARD' | 'STANDARD'): number {
 337   return VIDEO_TRANSCODING_FPS[type].slice(0)
 338                                     .sort((a, b) => fps % a - fps % b)[0]
 339 }
 340
 341 // ---------------------------------------------------------------------------
 342
 343 export {
 344   getVideoStreamCodec,
 345   getAudioStreamCodec,
 346   getVideoStreamSize,
 347   getVideoFileResolution,
 348   getMetadataFromFile,
 349   getDurationFromVideoFile,
 350   generateImageFromVideoFile,
 351   TranscodeOptions,
 352   TranscodeOptionsType,
 353   transcode,
 354   getVideoFileFPS,
 355   computeResolutionsToTranscode,
 356   audio,
 357   getVideoFileBitrate,
 358   canDoQuickTranscode
 359 }
 360
 361 // ---------------------------------------------------------------------------
 362
 363 async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 364   let fps = await getVideoFileFPS(options.inputPath)
 365   if (
 366     // On small/medium resolutions, limit FPS
 367     options.resolution !== undefined &&
 368     options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
 369     fps > VIDEO_TRANSCODING_FPS.AVERAGE
 370   ) {
 371     // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
 372     fps = getClosestFramerateStandard(fps, 'STANDARD')
 373   }
 374
 375   command = await presetH264(command, options.inputPath, options.resolution, fps)
 376
 377   if (options.resolution !== undefined) {
 378     // '?x720' or '720x?' for example
 379     const size = options.isPortraitMode === true ? `${options.resolution}x?` : `?x${options.resolution}`
 380     command = command.size(size)
 381   }
 382
 383   if (fps) {
 384     // Hard FPS limits
 385     if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
 386     else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
 387
 388     command = command.withFPS(fps)
 389   }
 390
 391   return command
 392 }
 393
 394 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 395   command = command.loop(undefined)
 396
 397   command = await presetH264VeryFast(command, options.audioPath, options.resolution)
 398
 399   command = command.input(options.audioPath)
 400                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 401                    .outputOption('-tune stillimage')
 402                    .outputOption('-shortest')
 403
 404   return command
 405 }
 406
 407 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) {
 408   command = presetOnlyAudio(command)
 409
 410   return command
 411 }
 412
 413 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 414   command = presetCopy(command)
 415
 416   command = command.outputOption('-map_metadata -1') // strip all metadata
 417                    .outputOption('-movflags faststart')
 418
 419   return command
 420 }
 421
 422 async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 423   const videoPath = getHLSVideoPath(options)
 424
 425   if (options.copyCodecs) command = presetCopy(command)
 426   else command = await buildx264Command(command, options)
 427
 428   command = command.outputOption('-hls_time 4')
 429                    .outputOption('-hls_list_size 0')
 430                    .outputOption('-hls_playlist_type vod')
 431                    .outputOption('-hls_segment_filename ' + videoPath)
 432                    .outputOption('-hls_segment_type fmp4')
 433                    .outputOption('-f hls')
 434                    .outputOption('-hls_flags single_file')
 435
 436   return command
 437 }
 438
 439 function getHLSVideoPath (options: HLSTranscodeOptions) {
 440   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 441 }
 442
 443 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 444   if (options.type !== 'hls') return
 445
 446   const fileContent = await readFile(options.outputPath)
 447
 448   const videoFileName = options.hlsPlaylist.videoFilename
 449   const videoFilePath = getHLSVideoPath(options)
 450
 451   // Fix wrong mapping with some ffmpeg versions
 452   const newContent = fileContent.toString()
 453                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 454
 455   await writeFile(options.outputPath, newContent)
 456 }
 457
 458 /**
 459  * A slightly customised version of the 'veryfast' x264 preset
 460  *
 461  * The veryfast preset is right in the sweet spot of performance
 462  * and quality. Superfast and ultrafast will give you better
 463  * performance, but then quality is noticeably worse.
 464  */
 465 async function presetH264VeryFast (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 466   let localCommand = await presetH264(command, input, resolution, fps)
 467
 468   localCommand = localCommand.outputOption('-preset:v veryfast')
 469
 470   /*
 471   MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html
 472   Our target situation is closer to a livestream than a stream,
 473   since we want to reduce as much a possible the encoding burden,
 474   although not to the point of a livestream where there is a hard
 475   constraint on the frames per second to be encoded.
 476   */
 477
 478   return localCommand
 479 }
 480
 481 /**
 482  * Standard profile, with variable bitrate audio and faststart.
 483  *
 484  * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel
 485  * See https://trac.ffmpeg.org/wiki/Encode/AAC#fdk_vbr
 486  */
 487 async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 488   let localCommand = command
 489     .format('mp4')
 490     .videoCodec('libx264')
 491     .outputOption('-level 3.1') // 3.1 is the minimal resource allocation for our highest supported resolution
 492     .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 493     .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 494     .outputOption('-pix_fmt yuv420p') // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 495     .outputOption('-map_metadata -1') // strip all metadata
 496     .outputOption('-movflags faststart')
 497
 498   const parsedAudio = await audio.get(input)
 499
 500   if (!parsedAudio.audioStream) {
 501     localCommand = localCommand.noAudio()
 502   } else if ((await checkFFmpegEncoders()).get('libfdk_aac')) { // we favor VBR, if a good AAC encoder is available
 503     localCommand = localCommand
 504       .audioCodec('libfdk_aac')
 505       .audioQuality(5)
 506   } else {
 507     // we try to reduce the ceiling bitrate by making rough matches of bitrates
 508     // of course this is far from perfect, but it might save some space in the end
 509     localCommand = localCommand.audioCodec('aac')
 510
 511     const audioCodecName = parsedAudio.audioStream['codec_name']
 512
 513     if (audio.bitrate[audioCodecName]) {
 514       const bitrate = audio.bitrate[audioCodecName](parsedAudio.audioStream['bit_rate'])
 515       if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate)
 516     }
 517   }
 518
 519   if (fps) {
 520     // Constrained Encoding (VBV)
 521     // https://slhck.info/video/2017/03/01/rate-control.html
 522     // https://trac.ffmpeg.org/wiki/Limiting%20the%20output%20bitrate
 523     const targetBitrate = getTargetBitrate(resolution, fps, VIDEO_TRANSCODING_FPS)
 524     localCommand = localCommand.outputOptions([ `-maxrate ${targetBitrate}`, `-bufsize ${targetBitrate * 2}` ])
 525
 526     // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 527     // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 528     // https://superuser.com/a/908325
 529     localCommand = localCommand.outputOption(`-g ${fps * 2}`)
 530   }
 531
 532   return localCommand
 533 }
 534
 535 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 536   return command
 537     .format('mp4')
 538     .videoCodec('copy')
 539     .audioCodec('copy')
 540 }
 541
 542 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 543   return command
 544     .format('mp4')
 545     .audioCodec('copy')
 546     .noVideo()
 547 }