server/helpers/ffmpeg-utils.ts

   1 import * as ffmpeg from 'fluent-ffmpeg'
   2 import { dirname, join } from 'path'
   3 import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos'
   4 import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
   5 import { processImage } from './image-utils'
   6 import { logger } from './logger'
   7 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   8 import { readFile, remove, writeFile } from 'fs-extra'
   9 import { CONFIG } from '../initializers/config'
  10 import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
  11
  12 /**
  13  * A toolbox to play with audio
  14  */
  15 namespace audio {
  16   export const get = (videoPath: string) => {
  17     // without position, ffprobe considers the last input only
  18     // we make it consider the first input only
  19     // if you pass a file path to pos, then ffprobe acts on that file directly
  20     return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
  21
  22       function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
  23         if (err) return rej(err)
  24
  25         if ('streams' in data) {
  26           const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
  27           if (audioStream) {
  28             return res({
  29               absolutePath: data.format.filename,
  30               audioStream
  31             })
  32           }
  33         }
  34
  35         return res({ absolutePath: data.format.filename })
  36       }
  37
  38       return ffmpeg.ffprobe(videoPath, parseFfprobe)
  39     })
  40   }
  41
  42   export namespace bitrate {
  43     const baseKbitrate = 384
  44
  45     const toBits = (kbits: number) => kbits * 8000
  46
  47     export const aac = (bitrate: number): number => {
  48       switch (true) {
  49         case bitrate > toBits(baseKbitrate):
  50           return baseKbitrate
  51
  52         default:
  53           return -1 // we interpret it as a signal to copy the audio stream as is
  54       }
  55     }
  56
  57     export const mp3 = (bitrate: number): number => {
  58       /*
  59       a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
  60       That's why, when using aac, we can go to lower kbit/sec. The equivalences
  61       made here are not made to be accurate, especially with good mp3 encoders.
  62       */
  63       switch (true) {
  64         case bitrate <= toBits(192):
  65           return 128
  66
  67         case bitrate <= toBits(384):
  68           return 256
  69
  70         default:
  71           return baseKbitrate
  72       }
  73     }
  74   }
  75 }
  76
  77 function computeResolutionsToTranscode (videoFileHeight: number) {
  78   const resolutionsEnabled: number[] = []
  79   const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
  80
  81   // Put in the order we want to proceed jobs
  82   const resolutions = [
  83     VideoResolution.H_NOVIDEO,
  84     VideoResolution.H_480P,
  85     VideoResolution.H_360P,
  86     VideoResolution.H_720P,
  87     VideoResolution.H_240P,
  88     VideoResolution.H_1080P,
  89     VideoResolution.H_4K
  90   ]
  91
  92   for (const resolution of resolutions) {
  93     if (configResolutions[resolution + 'p'] === true && videoFileHeight > resolution) {
  94       resolutionsEnabled.push(resolution)
  95     }
  96   }
  97
  98   return resolutionsEnabled
  99 }
 100
 101 async function getVideoStreamSize (path: string) {
 102   const videoStream = await getVideoStreamFromFile(path)
 103
 104   return videoStream === null
 105     ? { width: 0, height: 0 }
 106     : { width: videoStream.width, height: videoStream.height }
 107 }
 108
 109 async function getVideoStreamCodec (path: string) {
 110   const videoStream = await getVideoStreamFromFile(path)
 111
 112   if (!videoStream) return ''
 113
 114   const videoCodec = videoStream.codec_tag_string
 115
 116   const baseProfileMatrix = {
 117     High: '6400',
 118     Main: '4D40',
 119     Baseline: '42E0'
 120   }
 121
 122   let baseProfile = baseProfileMatrix[videoStream.profile]
 123   if (!baseProfile) {
 124     logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
 125     baseProfile = baseProfileMatrix['High'] // Fallback
 126   }
 127
 128   let level = videoStream.level.toString(16)
 129   if (level.length === 1) level = `0${level}`
 130
 131   return `${videoCodec}.${baseProfile}${level}`
 132 }
 133
 134 async function getAudioStreamCodec (path: string) {
 135   const { audioStream } = await audio.get(path)
 136
 137   if (!audioStream) return ''
 138
 139   const audioCodec = audioStream.codec_name
 140   if (audioCodec === 'aac') return 'mp4a.40.2'
 141
 142   logger.warn('Cannot get audio codec of %s.', path, { audioStream })
 143
 144   return 'mp4a.40.2' // Fallback
 145 }
 146
 147 async function getVideoFileResolution (path: string) {
 148   const size = await getVideoStreamSize(path)
 149
 150   return {
 151     videoFileResolution: Math.min(size.height, size.width),
 152     isPortraitMode: size.height > size.width
 153   }
 154 }
 155
 156 async function getVideoFileFPS (path: string) {
 157   const videoStream = await getVideoStreamFromFile(path)
 158   if (videoStream === null) return 0
 159
 160   for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
 161     const valuesText: string = videoStream[key]
 162     if (!valuesText) continue
 163
 164     const [ frames, seconds ] = valuesText.split('/')
 165     if (!frames || !seconds) continue
 166
 167     const result = parseInt(frames, 10) / parseInt(seconds, 10)
 168     if (result > 0) return Math.round(result)
 169   }
 170
 171   return 0
 172 }
 173
 174 async function getMetadataFromFile <T> (path: string, cb = metadata => metadata) {
 175   return new Promise<T>((res, rej) => {
 176     ffmpeg.ffprobe(path, (err, metadata) => {
 177       if (err) return rej(err)
 178
 179       return res(cb(new VideoFileMetadata(metadata)))
 180     })
 181   })
 182 }
 183
 184 async function getVideoFileBitrate (path: string) {
 185   return getMetadataFromFile<number>(path, metadata => metadata.format.bit_rate)
 186 }
 187
 188 function getDurationFromVideoFile (path: string) {
 189   return getMetadataFromFile<number>(path, metadata => Math.floor(metadata.format.duration))
 190 }
 191
 192 function getVideoStreamFromFile (path: string) {
 193   return getMetadataFromFile<any>(path, metadata => metadata.streams.find(s => s.codec_type === 'video') || null)
 194 }
 195
 196 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
 197   const pendingImageName = 'pending-' + imageName
 198
 199   const options = {
 200     filename: pendingImageName,
 201     count: 1,
 202     folder
 203   }
 204
 205   const pendingImagePath = join(folder, pendingImageName)
 206
 207   try {
 208     await new Promise<string>((res, rej) => {
 209       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
 210         .on('error', rej)
 211         .on('end', () => res(imageName))
 212         .thumbnail(options)
 213     })
 214
 215     const destination = join(folder, imageName)
 216     await processImage(pendingImagePath, destination, size)
 217   } catch (err) {
 218     logger.error('Cannot generate image from video %s.', fromPath, { err })
 219
 220     try {
 221       await remove(pendingImagePath)
 222     } catch (err) {
 223       logger.debug('Cannot remove pending image path after generation error.', { err })
 224     }
 225   }
 226 }
 227
 228 type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 229
 230 interface BaseTranscodeOptions {
 231   type: TranscodeOptionsType
 232   inputPath: string
 233   outputPath: string
 234   resolution: VideoResolution
 235   isPortraitMode?: boolean
 236 }
 237
 238 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 239   type: 'hls'
 240   copyCodecs: boolean
 241   hlsPlaylist: {
 242     videoFilename: string
 243   }
 244 }
 245
 246 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 247   type: 'quick-transcode'
 248 }
 249
 250 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 251   type: 'video'
 252 }
 253
 254 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 255   type: 'merge-audio'
 256   audioPath: string
 257 }
 258
 259 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 260   type: 'only-audio'
 261 }
 262
 263 type TranscodeOptions =
 264   HLSTranscodeOptions
 265   | VideoTranscodeOptions
 266   | MergeAudioTranscodeOptions
 267   | OnlyAudioTranscodeOptions
 268   | QuickTranscodeOptions
 269
 270 function transcode (options: TranscodeOptions) {
 271   return new Promise<void>(async (res, rej) => {
 272     try {
 273       let command = ffmpeg(options.inputPath, { niceness: FFMPEG_NICE.TRANSCODING })
 274         .output(options.outputPath)
 275
 276       if (options.type === 'quick-transcode') {
 277         command = buildQuickTranscodeCommand(command)
 278       } else if (options.type === 'hls') {
 279         command = await buildHLSCommand(command, options)
 280       } else if (options.type === 'merge-audio') {
 281         command = await buildAudioMergeCommand(command, options)
 282       } else if (options.type === 'only-audio') {
 283         command = buildOnlyAudioCommand(command, options)
 284       } else {
 285         command = await buildx264Command(command, options)
 286       }
 287
 288       if (CONFIG.TRANSCODING.THREADS > 0) {
 289         // if we don't set any threads ffmpeg will chose automatically
 290         command = command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
 291       }
 292
 293       command
 294         .on('error', (err, stdout, stderr) => {
 295           logger.error('Error in transcoding job.', { stdout, stderr })
 296           return rej(err)
 297         })
 298         .on('end', () => {
 299           return fixHLSPlaylistIfNeeded(options)
 300             .then(() => res())
 301             .catch(err => rej(err))
 302         })
 303         .run()
 304     } catch (err) {
 305       return rej(err)
 306     }
 307   })
 308 }
 309
 310 async function canDoQuickTranscode (path: string): Promise<boolean> {
 311   // NOTE: This could be optimized by running ffprobe only once (but it runs fast anyway)
 312   const videoStream = await getVideoStreamFromFile(path)
 313   const parsedAudio = await audio.get(path)
 314   const fps = await getVideoFileFPS(path)
 315   const bitRate = await getVideoFileBitrate(path)
 316   const resolution = await getVideoFileResolution(path)
 317
 318   // check video params
 319   if (videoStream == null) return false
 320   if (videoStream['codec_name'] !== 'h264') return false
 321   if (videoStream['pix_fmt'] !== 'yuv420p') return false
 322   if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
 323   if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false
 324
 325   // check audio params (if audio stream exists)
 326   if (parsedAudio.audioStream) {
 327     if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
 328
 329     const maxAudioBitrate = audio.bitrate['aac'](parsedAudio.audioStream['bit_rate'])
 330     if (maxAudioBitrate !== -1 && parsedAudio.audioStream['bit_rate'] > maxAudioBitrate) return false
 331   }
 332
 333   return true
 334 }
 335
 336 function getClosestFramerateStandard (fps: number, type: 'HD_STANDARD' | 'STANDARD'): number {
 337   return VIDEO_TRANSCODING_FPS[type].slice(0)
 338                                     .sort((a, b) => fps % a - fps % b)[0]
 339 }
 340
 341 function convertWebPToJPG (path: string, destination: string): Promise<void> {
 342   return new Promise<void>(async (res, rej) => {
 343     try {
 344       const command = ffmpeg(path).output(destination)
 345
 346       command.on('error', (err, stdout, stderr) => {
 347         logger.error('Error in ffmpeg webp convert process.', { stdout, stderr })
 348         return rej(err)
 349       })
 350       .on('end', () => res())
 351       .run()
 352     } catch (err) {
 353       return rej(err)
 354     }
 355   })
 356 }
 357
 358 // ---------------------------------------------------------------------------
 359
 360 export {
 361   getVideoStreamCodec,
 362   getAudioStreamCodec,
 363   convertWebPToJPG,
 364   getVideoStreamSize,
 365   getVideoFileResolution,
 366   getMetadataFromFile,
 367   getDurationFromVideoFile,
 368   generateImageFromVideoFile,
 369   TranscodeOptions,
 370   TranscodeOptionsType,
 371   transcode,
 372   getVideoFileFPS,
 373   computeResolutionsToTranscode,
 374   audio,
 375   getVideoFileBitrate,
 376   canDoQuickTranscode
 377 }
 378
 379 // ---------------------------------------------------------------------------
 380
 381 async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 382   let fps = await getVideoFileFPS(options.inputPath)
 383   if (
 384     // On small/medium resolutions, limit FPS
 385     options.resolution !== undefined &&
 386     options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
 387     fps > VIDEO_TRANSCODING_FPS.AVERAGE
 388   ) {
 389     // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
 390     fps = getClosestFramerateStandard(fps, 'STANDARD')
 391   }
 392
 393   command = await presetH264(command, options.inputPath, options.resolution, fps)
 394
 395   if (options.resolution !== undefined) {
 396     // '?x720' or '720x?' for example
 397     const size = options.isPortraitMode === true ? `${options.resolution}x?` : `?x${options.resolution}`
 398     command = command.size(size)
 399   }
 400
 401   if (fps) {
 402     // Hard FPS limits
 403     if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
 404     else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
 405
 406     command = command.withFPS(fps)
 407   }
 408
 409   return command
 410 }
 411
 412 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 413   command = command.loop(undefined)
 414
 415   command = await presetH264VeryFast(command, options.audioPath, options.resolution)
 416
 417   command = command.input(options.audioPath)
 418                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 419                    .outputOption('-tune stillimage')
 420                    .outputOption('-shortest')
 421
 422   return command
 423 }
 424
 425 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) {
 426   command = presetOnlyAudio(command)
 427
 428   return command
 429 }
 430
 431 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 432   command = presetCopy(command)
 433
 434   command = command.outputOption('-map_metadata -1') // strip all metadata
 435                    .outputOption('-movflags faststart')
 436
 437   return command
 438 }
 439
 440 async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 441   const videoPath = getHLSVideoPath(options)
 442
 443   if (options.copyCodecs) command = presetCopy(command)
 444   else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
 445   else command = await buildx264Command(command, options)
 446
 447   command = command.outputOption('-hls_time 4')
 448                    .outputOption('-hls_list_size 0')
 449                    .outputOption('-hls_playlist_type vod')
 450                    .outputOption('-hls_segment_filename ' + videoPath)
 451                    .outputOption('-hls_segment_type fmp4')
 452                    .outputOption('-f hls')
 453                    .outputOption('-hls_flags single_file')
 454
 455   return command
 456 }
 457
 458 function getHLSVideoPath (options: HLSTranscodeOptions) {
 459   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 460 }
 461
 462 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 463   if (options.type !== 'hls') return
 464
 465   const fileContent = await readFile(options.outputPath)
 466
 467   const videoFileName = options.hlsPlaylist.videoFilename
 468   const videoFilePath = getHLSVideoPath(options)
 469
 470   // Fix wrong mapping with some ffmpeg versions
 471   const newContent = fileContent.toString()
 472                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 473
 474   await writeFile(options.outputPath, newContent)
 475 }
 476
 477 /**
 478  * A slightly customised version of the 'veryfast' x264 preset
 479  *
 480  * The veryfast preset is right in the sweet spot of performance
 481  * and quality. Superfast and ultrafast will give you better
 482  * performance, but then quality is noticeably worse.
 483  */
 484 async function presetH264VeryFast (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 485   let localCommand = await presetH264(command, input, resolution, fps)
 486
 487   localCommand = localCommand.outputOption('-preset:v veryfast')
 488
 489   /*
 490   MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html
 491   Our target situation is closer to a livestream than a stream,
 492   since we want to reduce as much a possible the encoding burden,
 493   although not to the point of a livestream where there is a hard
 494   constraint on the frames per second to be encoded.
 495   */
 496
 497   return localCommand
 498 }
 499
 500 /**
 501  * Standard profile, with variable bitrate audio and faststart.
 502  *
 503  * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel
 504  * See https://trac.ffmpeg.org/wiki/Encode/AAC#fdk_vbr
 505  */
 506 async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 507   let localCommand = command
 508     .format('mp4')
 509     .videoCodec('libx264')
 510     .outputOption('-level 3.1') // 3.1 is the minimal resource allocation for our highest supported resolution
 511     .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 512     .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 513     .outputOption('-pix_fmt yuv420p') // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 514     .outputOption('-map_metadata -1') // strip all metadata
 515     .outputOption('-movflags faststart')
 516
 517   const parsedAudio = await audio.get(input)
 518
 519   if (!parsedAudio.audioStream) {
 520     localCommand = localCommand.noAudio()
 521   } else if ((await checkFFmpegEncoders()).get('libfdk_aac')) { // we favor VBR, if a good AAC encoder is available
 522     localCommand = localCommand
 523       .audioCodec('libfdk_aac')
 524       .audioQuality(5)
 525   } else {
 526     // we try to reduce the ceiling bitrate by making rough matches of bitrates
 527     // of course this is far from perfect, but it might save some space in the end
 528     localCommand = localCommand.audioCodec('aac')
 529
 530     const audioCodecName = parsedAudio.audioStream['codec_name']
 531
 532     if (audio.bitrate[audioCodecName]) {
 533       const bitrate = audio.bitrate[audioCodecName](parsedAudio.audioStream['bit_rate'])
 534       if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate)
 535     }
 536   }
 537
 538   if (fps) {
 539     // Constrained Encoding (VBV)
 540     // https://slhck.info/video/2017/03/01/rate-control.html
 541     // https://trac.ffmpeg.org/wiki/Limiting%20the%20output%20bitrate
 542     const targetBitrate = getTargetBitrate(resolution, fps, VIDEO_TRANSCODING_FPS)
 543     localCommand = localCommand.outputOptions([ `-maxrate ${targetBitrate}`, `-bufsize ${targetBitrate * 2}` ])
 544
 545     // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 546     // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 547     // https://superuser.com/a/908325
 548     localCommand = localCommand.outputOption(`-g ${fps * 2}`)
 549   }
 550
 551   return localCommand
 552 }
 553
 554 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 555   return command
 556     .format('mp4')
 557     .videoCodec('copy')
 558     .audioCodec('copy')
 559 }
 560
 561 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 562   return command
 563     .format('mp4')
 564     .audioCodec('copy')
 565     .noVideo()
 566 }