server/helpers/ffmpeg-utils.ts

   1 import * as ffmpeg from 'fluent-ffmpeg'
   2 import { dirname, join } from 'path'
   3 import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos'
   4 import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
   5 import { processImage } from './image-utils'
   6 import { logger } from './logger'
   7 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   8 import { readFile, remove, writeFile } from 'fs-extra'
   9 import { CONFIG } from '../initializers/config'
  10 import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
  11
  12 /**
  13  * A toolbox to play with audio
  14  */
  15 namespace audio {
  16   export const get = (videoPath: string) => {
  17     // without position, ffprobe considers the last input only
  18     // we make it consider the first input only
  19     // if you pass a file path to pos, then ffprobe acts on that file directly
  20     return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
  21
  22       function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
  23         if (err) return rej(err)
  24
  25         if ('streams' in data) {
  26           const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
  27           if (audioStream) {
  28             return res({
  29               absolutePath: data.format.filename,
  30               audioStream
  31             })
  32           }
  33         }
  34
  35         return res({ absolutePath: data.format.filename })
  36       }
  37
  38       return ffmpeg.ffprobe(videoPath, parseFfprobe)
  39     })
  40   }
  41
  42   export namespace bitrate {
  43     const baseKbitrate = 384
  44
  45     const toBits = (kbits: number) => kbits * 8000
  46
  47     export const aac = (bitrate: number): number => {
  48       switch (true) {
  49         case bitrate > toBits(baseKbitrate):
  50           return baseKbitrate
  51
  52         default:
  53           return -1 // we interpret it as a signal to copy the audio stream as is
  54       }
  55     }
  56
  57     export const mp3 = (bitrate: number): number => {
  58       /*
  59       a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
  60       That's why, when using aac, we can go to lower kbit/sec. The equivalences
  61       made here are not made to be accurate, especially with good mp3 encoders.
  62       */
  63       switch (true) {
  64         case bitrate <= toBits(192):
  65           return 128
  66
  67         case bitrate <= toBits(384):
  68           return 256
  69
  70         default:
  71           return baseKbitrate
  72       }
  73     }
  74   }
  75 }
  76
  77 function computeResolutionsToTranscode (videoFileHeight: number) {
  78   const resolutionsEnabled: number[] = []
  79   const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
  80
  81   // Put in the order we want to proceed jobs
  82   const resolutions = [
  83     VideoResolution.H_NOVIDEO,
  84     VideoResolution.H_480P,
  85     VideoResolution.H_360P,
  86     VideoResolution.H_720P,
  87     VideoResolution.H_240P,
  88     VideoResolution.H_1080P,
  89     VideoResolution.H_4K
  90   ]
  91
  92   for (const resolution of resolutions) {
  93     if (configResolutions[resolution + 'p'] === true && videoFileHeight > resolution) {
  94       resolutionsEnabled.push(resolution)
  95     }
  96   }
  97
  98   return resolutionsEnabled
  99 }
 100
 101 async function getVideoStreamSize (path: string) {
 102   const videoStream = await getVideoStreamFromFile(path)
 103
 104   return videoStream === null
 105     ? { width: 0, height: 0 }
 106     : { width: videoStream.width, height: videoStream.height }
 107 }
 108
 109 async function getVideoStreamCodec (path: string) {
 110   const videoStream = await getVideoStreamFromFile(path)
 111
 112   if (!videoStream) return ''
 113
 114   const videoCodec = videoStream.codec_tag_string
 115
 116   const baseProfileMatrix = {
 117     High: '6400',
 118     Main: '4D40',
 119     Baseline: '42E0'
 120   }
 121
 122   let baseProfile = baseProfileMatrix[videoStream.profile]
 123   if (!baseProfile) {
 124     logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
 125     baseProfile = baseProfileMatrix['High'] // Fallback
 126   }
 127
 128   let level = videoStream.level.toString(16)
 129   if (level.length === 1) level = `0${level}`
 130
 131   return `${videoCodec}.${baseProfile}${level}`
 132 }
 133
 134 async function getAudioStreamCodec (path: string) {
 135   const { audioStream } = await audio.get(path)
 136
 137   if (!audioStream) return ''
 138
 139   const audioCodec = audioStream.codec_name
 140   if (audioCodec === 'aac') return 'mp4a.40.2'
 141
 142   logger.warn('Cannot get audio codec of %s.', path, { audioStream })
 143
 144   return 'mp4a.40.2' // Fallback
 145 }
 146
 147 async function getVideoFileResolution (path: string) {
 148   const size = await getVideoStreamSize(path)
 149
 150   return {
 151     videoFileResolution: Math.min(size.height, size.width),
 152     isPortraitMode: size.height > size.width
 153   }
 154 }
 155
 156 async function getVideoFileFPS (path: string) {
 157   const videoStream = await getVideoStreamFromFile(path)
 158   if (videoStream === null) return 0
 159
 160   for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
 161     const valuesText: string = videoStream[key]
 162     if (!valuesText) continue
 163
 164     const [ frames, seconds ] = valuesText.split('/')
 165     if (!frames || !seconds) continue
 166
 167     const result = parseInt(frames, 10) / parseInt(seconds, 10)
 168     if (result > 0) return Math.round(result)
 169   }
 170
 171   return 0
 172 }
 173
 174 async function getMetadataFromFile <T> (path: string, cb = metadata => metadata) {
 175   return new Promise<T>((res, rej) => {
 176     ffmpeg.ffprobe(path, (err, metadata) => {
 177       if (err) return rej(err)
 178
 179       return res(cb(new VideoFileMetadata(metadata)))
 180     })
 181   })
 182 }
 183
 184 async function getVideoFileBitrate (path: string) {
 185   return getMetadataFromFile<number>(path, metadata => metadata.format.bit_rate)
 186 }
 187
 188 function getDurationFromVideoFile (path: string) {
 189   return getMetadataFromFile<number>(path, metadata => Math.floor(metadata.format.duration))
 190 }
 191
 192 function getVideoStreamFromFile (path: string) {
 193   return getMetadataFromFile<any>(path, metadata => metadata.streams.find(s => s.codec_type === 'video') || null)
 194 }
 195
 196 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
 197   const pendingImageName = 'pending-' + imageName
 198
 199   const options = {
 200     filename: pendingImageName,
 201     count: 1,
 202     folder
 203   }
 204
 205   const pendingImagePath = join(folder, pendingImageName)
 206
 207   try {
 208     await new Promise<string>((res, rej) => {
 209       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
 210         .on('error', rej)
 211         .on('end', () => res(imageName))
 212         .thumbnail(options)
 213     })
 214
 215     const destination = join(folder, imageName)
 216     await processImage(pendingImagePath, destination, size)
 217   } catch (err) {
 218     logger.error('Cannot generate image from video %s.', fromPath, { err })
 219
 220     try {
 221       await remove(pendingImagePath)
 222     } catch (err) {
 223       logger.debug('Cannot remove pending image path after generation error.', { err })
 224     }
 225   }
 226 }
 227
 228 type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 229
 230 interface BaseTranscodeOptions {
 231   type: TranscodeOptionsType
 232   inputPath: string
 233   outputPath: string
 234   resolution: VideoResolution
 235   isPortraitMode?: boolean
 236 }
 237
 238 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 239   type: 'hls'
 240   copyCodecs: boolean
 241   hlsPlaylist: {
 242     videoFilename: string
 243   }
 244 }
 245
 246 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 247   type: 'quick-transcode'
 248 }
 249
 250 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 251   type: 'video'
 252 }
 253
 254 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 255   type: 'merge-audio'
 256   audioPath: string
 257 }
 258
 259 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 260   type: 'only-audio'
 261 }
 262
 263 type TranscodeOptions =
 264   HLSTranscodeOptions
 265   | VideoTranscodeOptions
 266   | MergeAudioTranscodeOptions
 267   | OnlyAudioTranscodeOptions
 268   | QuickTranscodeOptions
 269
 270 function transcode (options: TranscodeOptions) {
 271   return new Promise<void>(async (res, rej) => {
 272     try {
 273       let command = ffmpeg(options.inputPath, { niceness: FFMPEG_NICE.TRANSCODING })
 274         .output(options.outputPath)
 275
 276       if (options.type === 'quick-transcode') {
 277         command = buildQuickTranscodeCommand(command)
 278       } else if (options.type === 'hls') {
 279         command = await buildHLSCommand(command, options)
 280       } else if (options.type === 'merge-audio') {
 281         command = await buildAudioMergeCommand(command, options)
 282       } else if (options.type === 'only-audio') {
 283         command = buildOnlyAudioCommand(command, options)
 284       } else {
 285         command = await buildx264Command(command, options)
 286       }
 287
 288       if (CONFIG.TRANSCODING.THREADS > 0) {
 289         // if we don't set any threads ffmpeg will chose automatically
 290         command = command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
 291       }
 292
 293       command
 294         .on('error', (err, stdout, stderr) => {
 295           logger.error('Error in transcoding job.', { stdout, stderr })
 296           return rej(err)
 297         })
 298         .on('end', () => {
 299           return fixHLSPlaylistIfNeeded(options)
 300             .then(() => res())
 301             .catch(err => rej(err))
 302         })
 303         .run()
 304     } catch (err) {
 305       return rej(err)
 306     }
 307   })
 308 }
 309
 310 async function canDoQuickTranscode (path: string): Promise<boolean> {
 311   // NOTE: This could be optimized by running ffprobe only once (but it runs fast anyway)
 312   const videoStream = await getVideoStreamFromFile(path)
 313   const parsedAudio = await audio.get(path)
 314   const fps = await getVideoFileFPS(path)
 315   const bitRate = await getVideoFileBitrate(path)
 316   const resolution = await getVideoFileResolution(path)
 317
 318   // check video params
 319   if (videoStream == null) return false
 320   if (videoStream['codec_name'] !== 'h264') return false
 321   if (videoStream['pix_fmt'] !== 'yuv420p') return false
 322   if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
 323   if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false
 324
 325   // check audio params (if audio stream exists)
 326   if (parsedAudio.audioStream) {
 327     if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
 328
 329     const maxAudioBitrate = audio.bitrate['aac'](parsedAudio.audioStream['bit_rate'])
 330     if (maxAudioBitrate !== -1 && parsedAudio.audioStream['bit_rate'] > maxAudioBitrate) return false
 331   }
 332
 333   return true
 334 }
 335
 336 function getClosestFramerateStandard (fps: number, type: 'HD_STANDARD' | 'STANDARD'): number {
 337   return VIDEO_TRANSCODING_FPS[type].slice(0)
 338                                     .sort((a, b) => fps % a - fps % b)[0]
 339 }
 340
 341 // ---------------------------------------------------------------------------
 342
 343 export {
 344   getVideoStreamCodec,
 345   getAudioStreamCodec,
 346   getVideoStreamSize,
 347   getVideoFileResolution,
 348   getMetadataFromFile,
 349   getDurationFromVideoFile,
 350   generateImageFromVideoFile,
 351   TranscodeOptions,
 352   TranscodeOptionsType,
 353   transcode,
 354   getVideoFileFPS,
 355   computeResolutionsToTranscode,
 356   audio,
 357   getVideoFileBitrate,
 358   canDoQuickTranscode
 359 }
 360
 361 // ---------------------------------------------------------------------------
 362
 363 async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 364   let fps = await getVideoFileFPS(options.inputPath)
 365   if (
 366     // On small/medium resolutions, limit FPS
 367     options.resolution !== undefined &&
 368     options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
 369     fps > VIDEO_TRANSCODING_FPS.AVERAGE
 370   ) {
 371     // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
 372     fps = getClosestFramerateStandard(fps, 'STANDARD')
 373   }
 374
 375   command = await presetH264(command, options.inputPath, options.resolution, fps)
 376
 377   if (options.resolution !== undefined) {
 378     // '?x720' or '720x?' for example
 379     const size = options.isPortraitMode === true ? `${options.resolution}x?` : `?x${options.resolution}`
 380     command = command.size(size)
 381   }
 382
 383   if (fps) {
 384     // Hard FPS limits
 385     if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
 386     else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
 387
 388     command = command.withFPS(fps)
 389   }
 390
 391   return command
 392 }
 393
 394 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 395   command = command.loop(undefined)
 396
 397   command = await presetH264VeryFast(command, options.audioPath, options.resolution)
 398
 399   command = command.input(options.audioPath)
 400                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 401                    .outputOption('-tune stillimage')
 402                    .outputOption('-shortest')
 403
 404   return command
 405 }
 406
 407 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) {
 408   command = presetOnlyAudio(command)
 409
 410   return command
 411 }
 412
 413 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 414   command = presetCopy(command)
 415
 416   command = command.outputOption('-map_metadata -1') // strip all metadata
 417                    .outputOption('-movflags faststart')
 418
 419   return command
 420 }
 421
 422 async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 423   const videoPath = getHLSVideoPath(options)
 424
 425   if (options.copyCodecs) command = presetCopy(command)
 426   else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
 427   else command = await buildx264Command(command, options)
 428
 429   command = command.outputOption('-hls_time 4')
 430                    .outputOption('-hls_list_size 0')
 431                    .outputOption('-hls_playlist_type vod')
 432                    .outputOption('-hls_segment_filename ' + videoPath)
 433                    .outputOption('-hls_segment_type fmp4')
 434                    .outputOption('-f hls')
 435                    .outputOption('-hls_flags single_file')
 436
 437   return command
 438 }
 439
 440 function getHLSVideoPath (options: HLSTranscodeOptions) {
 441   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 442 }
 443
 444 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 445   if (options.type !== 'hls') return
 446
 447   const fileContent = await readFile(options.outputPath)
 448
 449   const videoFileName = options.hlsPlaylist.videoFilename
 450   const videoFilePath = getHLSVideoPath(options)
 451
 452   // Fix wrong mapping with some ffmpeg versions
 453   const newContent = fileContent.toString()
 454                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 455
 456   await writeFile(options.outputPath, newContent)
 457 }
 458
 459 /**
 460  * A slightly customised version of the 'veryfast' x264 preset
 461  *
 462  * The veryfast preset is right in the sweet spot of performance
 463  * and quality. Superfast and ultrafast will give you better
 464  * performance, but then quality is noticeably worse.
 465  */
 466 async function presetH264VeryFast (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 467   let localCommand = await presetH264(command, input, resolution, fps)
 468
 469   localCommand = localCommand.outputOption('-preset:v veryfast')
 470
 471   /*
 472   MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html
 473   Our target situation is closer to a livestream than a stream,
 474   since we want to reduce as much a possible the encoding burden,
 475   although not to the point of a livestream where there is a hard
 476   constraint on the frames per second to be encoded.
 477   */
 478
 479   return localCommand
 480 }
 481
 482 /**
 483  * Standard profile, with variable bitrate audio and faststart.
 484  *
 485  * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel
 486  * See https://trac.ffmpeg.org/wiki/Encode/AAC#fdk_vbr
 487  */
 488 async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 489   let localCommand = command
 490     .format('mp4')
 491     .videoCodec('libx264')
 492     .outputOption('-level 3.1') // 3.1 is the minimal resource allocation for our highest supported resolution
 493     .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 494     .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 495     .outputOption('-pix_fmt yuv420p') // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 496     .outputOption('-map_metadata -1') // strip all metadata
 497     .outputOption('-movflags faststart')
 498
 499   const parsedAudio = await audio.get(input)
 500
 501   if (!parsedAudio.audioStream) {
 502     localCommand = localCommand.noAudio()
 503   } else if ((await checkFFmpegEncoders()).get('libfdk_aac')) { // we favor VBR, if a good AAC encoder is available
 504     localCommand = localCommand
 505       .audioCodec('libfdk_aac')
 506       .audioQuality(5)
 507   } else {
 508     // we try to reduce the ceiling bitrate by making rough matches of bitrates
 509     // of course this is far from perfect, but it might save some space in the end
 510     localCommand = localCommand.audioCodec('aac')
 511
 512     const audioCodecName = parsedAudio.audioStream['codec_name']
 513
 514     if (audio.bitrate[audioCodecName]) {
 515       const bitrate = audio.bitrate[audioCodecName](parsedAudio.audioStream['bit_rate'])
 516       if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate)
 517     }
 518   }
 519
 520   if (fps) {
 521     // Constrained Encoding (VBV)
 522     // https://slhck.info/video/2017/03/01/rate-control.html
 523     // https://trac.ffmpeg.org/wiki/Limiting%20the%20output%20bitrate
 524     const targetBitrate = getTargetBitrate(resolution, fps, VIDEO_TRANSCODING_FPS)
 525     localCommand = localCommand.outputOptions([ `-maxrate ${targetBitrate}`, `-bufsize ${targetBitrate * 2}` ])
 526
 527     // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 528     // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 529     // https://superuser.com/a/908325
 530     localCommand = localCommand.outputOption(`-g ${fps * 2}`)
 531   }
 532
 533   return localCommand
 534 }
 535
 536 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 537   return command
 538     .format('mp4')
 539     .videoCodec('copy')
 540     .audioCodec('copy')
 541 }
 542
 543 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 544   return command
 545     .format('mp4')
 546     .audioCodec('copy')
 547     .noVideo()
 548 }