server/helpers/ffmpeg-utils.ts

   1 import * as ffmpeg from 'fluent-ffmpeg'
   2 import { readFile, remove, writeFile } from 'fs-extra'
   3 import { dirname, join } from 'path'
   4 import { FFMPEG_NICE, VIDEO_LIVE, VIDEO_TRANSCODING_ENCODERS, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
   5 import { VideoResolution } from '../../shared/models/videos'
   6 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   7 import { CONFIG } from '../initializers/config'
   8 import { getAudioStream, getClosestFramerateStandard, getVideoFileFPS } from './ffprobe-utils'
   9 import { processImage } from './image-utils'
  10 import { logger } from './logger'
  11
  12 // ---------------------------------------------------------------------------
  13 // Encoder options
  14 // ---------------------------------------------------------------------------
  15
  16 // Options builders
  17
  18 export type EncoderOptionsBuilder = (params: {
  19   input: string
  20   resolution: VideoResolution
  21   fps?: number
  22   streamNum?: number
  23 }) => Promise<EncoderOptions> | EncoderOptions
  24
  25 // Options types
  26
  27 export interface EncoderOptions {
  28   copy?: boolean
  29   outputOptions: string[]
  30 }
  31
  32 // All our encoders
  33
  34 export interface EncoderProfile <T> {
  35   [ profile: string ]: T
  36
  37   default: T
  38 }
  39
  40 export type AvailableEncoders = {
  41   [ id in 'live' | 'vod' ]: {
  42     [ encoder in 'libx264' | 'aac' | 'libfdk_aac' ]?: EncoderProfile<EncoderOptionsBuilder>
  43   }
  44 }
  45
  46 // ---------------------------------------------------------------------------
  47 // Image manipulation
  48 // ---------------------------------------------------------------------------
  49
  50 function convertWebPToJPG (path: string, destination: string): Promise<void> {
  51   const command = ffmpeg(path)
  52     .output(destination)
  53
  54   return runCommand(command)
  55 }
  56
  57 function processGIF (
  58   path: string,
  59   destination: string,
  60   newSize: { width: number, height: number },
  61   keepOriginal = false
  62 ): Promise<void> {
  63   return new Promise<void>(async (res, rej) => {
  64     if (path === destination) {
  65       throw new Error('FFmpeg needs an input path different that the output path.')
  66     }
  67
  68     logger.debug('Processing gif %s to %s.', path, destination)
  69
  70     try {
  71       const command = ffmpeg(path)
  72         .fps(20)
  73         .size(`${newSize.width}x${newSize.height}`)
  74         .output(destination)
  75
  76       command.on('error', (err, stdout, stderr) => {
  77         logger.error('Error in ffmpeg gif resizing process.', { stdout, stderr })
  78         return rej(err)
  79       })
  80       .on('end', async () => {
  81         if (keepOriginal !== true) await remove(path)
  82         res()
  83       })
  84       .run()
  85     } catch (err) {
  86       return rej(err)
  87     }
  88   })
  89 }
  90
  91 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
  92   const pendingImageName = 'pending-' + imageName
  93
  94   const options = {
  95     filename: pendingImageName,
  96     count: 1,
  97     folder
  98   }
  99
 100   const pendingImagePath = join(folder, pendingImageName)
 101
 102   try {
 103     await new Promise<string>((res, rej) => {
 104       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
 105         .on('error', rej)
 106         .on('end', () => res(imageName))
 107         .thumbnail(options)
 108     })
 109
 110     const destination = join(folder, imageName)
 111     await processImage(pendingImagePath, destination, size)
 112   } catch (err) {
 113     logger.error('Cannot generate image from video %s.', fromPath, { err })
 114
 115     try {
 116       await remove(pendingImagePath)
 117     } catch (err) {
 118       logger.debug('Cannot remove pending image path after generation error.', { err })
 119     }
 120   }
 121 }
 122
 123 // ---------------------------------------------------------------------------
 124 // Transcode meta function
 125 // ---------------------------------------------------------------------------
 126
 127 type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 128
 129 interface BaseTranscodeOptions {
 130   type: TranscodeOptionsType
 131
 132   inputPath: string
 133   outputPath: string
 134
 135   availableEncoders: AvailableEncoders
 136   profile: string
 137
 138   resolution: VideoResolution
 139
 140   isPortraitMode?: boolean
 141 }
 142
 143 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 144   type: 'hls'
 145   copyCodecs: boolean
 146   hlsPlaylist: {
 147     videoFilename: string
 148   }
 149 }
 150
 151 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 152   type: 'quick-transcode'
 153 }
 154
 155 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 156   type: 'video'
 157 }
 158
 159 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 160   type: 'merge-audio'
 161   audioPath: string
 162 }
 163
 164 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 165   type: 'only-audio'
 166 }
 167
 168 type TranscodeOptions =
 169   HLSTranscodeOptions
 170   | VideoTranscodeOptions
 171   | MergeAudioTranscodeOptions
 172   | OnlyAudioTranscodeOptions
 173   | QuickTranscodeOptions
 174
 175 const builders: {
 176   [ type in TranscodeOptionsType ]: (c: ffmpeg.FfmpegCommand, o?: TranscodeOptions) => Promise<ffmpeg.FfmpegCommand> | ffmpeg.FfmpegCommand
 177 } = {
 178   'quick-transcode': buildQuickTranscodeCommand,
 179   'hls': buildHLSVODCommand,
 180   'merge-audio': buildAudioMergeCommand,
 181   'only-audio': buildOnlyAudioCommand,
 182   'video': buildx264VODCommand
 183 }
 184
 185 async function transcode (options: TranscodeOptions) {
 186   logger.debug('Will run transcode.', { options })
 187
 188   let command = getFFmpeg(options.inputPath)
 189     .output(options.outputPath)
 190
 191   command = await builders[options.type](command, options)
 192
 193   await runCommand(command)
 194
 195   await fixHLSPlaylistIfNeeded(options)
 196 }
 197
 198 // ---------------------------------------------------------------------------
 199 // Live muxing/transcoding functions
 200 // ---------------------------------------------------------------------------
 201
 202 async function getLiveTranscodingCommand (options: {
 203   rtmpUrl: string
 204   outPath: string
 205   resolutions: number[]
 206   fps: number
 207   deleteSegments: boolean
 208
 209   availableEncoders: AvailableEncoders
 210   profile: string
 211 }) {
 212   const { rtmpUrl, outPath, resolutions, fps, deleteSegments, availableEncoders, profile } = options
 213   const input = rtmpUrl
 214
 215   const command = getFFmpeg(input)
 216   command.inputOption('-fflags nobuffer')
 217
 218   const varStreamMap: string[] = []
 219
 220   command.complexFilter([
 221     {
 222       inputs: '[v:0]',
 223       filter: 'split',
 224       options: resolutions.length,
 225       outputs: resolutions.map(r => `vtemp${r}`)
 226     },
 227
 228     ...resolutions.map(r => ({
 229       inputs: `vtemp${r}`,
 230       filter: 'scale',
 231       options: `w=-2:h=${r}`,
 232       outputs: `vout${r}`
 233     }))
 234   ])
 235
 236   command.outputOption('-preset superfast')
 237
 238   for (let i = 0; i < resolutions.length; i++) {
 239     const resolution = resolutions[i]
 240     const baseEncoderBuilderParams = { input, availableEncoders, profile, fps, resolution, streamNum: i, videoType: 'live' as 'live' }
 241
 242     {
 243       const builderResult = await getEncoderBuilderResult(Object.assign({}, baseEncoderBuilderParams, { streamType: 'VIDEO' }))
 244       if (!builderResult) {
 245         throw new Error('No available live video encoder found')
 246       }
 247
 248       command.outputOption(`-map [vout${resolution}]`)
 249
 250       addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i })
 251
 252       logger.debug('Apply ffmpeg live video params from %s.', builderResult.encoder, builderResult)
 253
 254       command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`)
 255       command.addOutputOptions(builderResult.result.outputOptions)
 256     }
 257
 258     {
 259       const builderResult = await getEncoderBuilderResult(Object.assign({}, baseEncoderBuilderParams, { streamType: 'AUDIO' }))
 260       if (!builderResult) {
 261         throw new Error('No available live audio encoder found')
 262       }
 263
 264       command.outputOption('-map a:0')
 265
 266       addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i })
 267
 268       logger.debug('Apply ffmpeg live audio params from %s.', builderResult.encoder, builderResult)
 269
 270       command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`)
 271       command.addOutputOptions(builderResult.result.outputOptions)
 272     }
 273
 274     varStreamMap.push(`v:${i},a:${i}`)
 275   }
 276
 277   addDefaultLiveHLSParams(command, outPath, deleteSegments)
 278
 279   command.outputOption('-var_stream_map', varStreamMap.join(' '))
 280
 281   return command
 282 }
 283
 284 function getLiveMuxingCommand (rtmpUrl: string, outPath: string, deleteSegments: boolean) {
 285   const command = getFFmpeg(rtmpUrl)
 286   command.inputOption('-fflags nobuffer')
 287
 288   command.outputOption('-c:v copy')
 289   command.outputOption('-c:a copy')
 290   command.outputOption('-map 0:a?')
 291   command.outputOption('-map 0:v?')
 292
 293   addDefaultLiveHLSParams(command, outPath, deleteSegments)
 294
 295   return command
 296 }
 297
 298 async function hlsPlaylistToFragmentedMP4 (hlsDirectory: string, segmentFiles: string[], outputPath: string) {
 299   const concatFilePath = join(hlsDirectory, 'concat.txt')
 300
 301   function cleaner () {
 302     remove(concatFilePath)
 303       .catch(err => logger.error('Cannot remove concat file in %s.', hlsDirectory, { err }))
 304   }
 305
 306   // First concat the ts files to a mp4 file
 307   const content = segmentFiles.map(f => 'file ' + f)
 308                               .join('\n')
 309
 310   await writeFile(concatFilePath, content + '\n')
 311
 312   const command = getFFmpeg(concatFilePath)
 313   command.inputOption('-safe 0')
 314   command.inputOption('-f concat')
 315
 316   command.outputOption('-c:v copy')
 317   command.audioFilter('aresample=async=1:first_pts=0')
 318   command.output(outputPath)
 319
 320   return runCommand(command, cleaner)
 321 }
 322
 323 function buildStreamSuffix (base: string, streamNum?: number) {
 324   if (streamNum !== undefined) {
 325     return `${base}:${streamNum}`
 326   }
 327
 328   return base
 329 }
 330
 331 // ---------------------------------------------------------------------------
 332
 333 export {
 334   getLiveTranscodingCommand,
 335   getLiveMuxingCommand,
 336   buildStreamSuffix,
 337   convertWebPToJPG,
 338   processGIF,
 339   generateImageFromVideoFile,
 340   TranscodeOptions,
 341   TranscodeOptionsType,
 342   transcode,
 343   hlsPlaylistToFragmentedMP4
 344 }
 345
 346 // ---------------------------------------------------------------------------
 347
 348 // ---------------------------------------------------------------------------
 349 // Default options
 350 // ---------------------------------------------------------------------------
 351
 352 function addDefaultEncoderParams (options: {
 353   command: ffmpeg.FfmpegCommand
 354   encoder: 'libx264' | string
 355   streamNum?: number
 356   fps?: number
 357 }) {
 358   const { command, encoder, fps, streamNum } = options
 359
 360   if (encoder === 'libx264') {
 361     // 3.1 is the minimal resource allocation for our highest supported resolution
 362     command.outputOption('-level 3.1')
 363         // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 364         .outputOption('-b_strategy 1')
 365         // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 366         .outputOption('-bf 16')
 367         // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 368         .outputOption(buildStreamSuffix('-pix_fmt', streamNum) + ' yuv420p')
 369         // strip all metadata
 370         .outputOption('-map_metadata -1')
 371         // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375
 372         .outputOption(buildStreamSuffix('-max_muxing_queue_size', streamNum) + ' 1024')
 373
 374     if (fps) {
 375       // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 376       // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 377       // https://superuser.com/a/908325
 378       command.outputOption('-g ' + (fps * 2))
 379     }
 380   }
 381 }
 382
 383 function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, deleteSegments: boolean) {
 384   command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS)
 385   command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
 386
 387   if (deleteSegments === true) {
 388     command.outputOption('-hls_flags delete_segments')
 389   }
 390
 391   command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
 392   command.outputOption('-master_pl_name master.m3u8')
 393   command.outputOption(`-f hls`)
 394
 395   command.output(join(outPath, '%v.m3u8'))
 396 }
 397
 398 // ---------------------------------------------------------------------------
 399 // Transcode VOD command builders
 400 // ---------------------------------------------------------------------------
 401
 402 async function buildx264VODCommand (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 403   let fps = await getVideoFileFPS(options.inputPath)
 404   if (
 405     // On small/medium resolutions, limit FPS
 406     options.resolution !== undefined &&
 407     options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
 408     fps > VIDEO_TRANSCODING_FPS.AVERAGE
 409   ) {
 410     // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
 411     fps = getClosestFramerateStandard(fps, 'STANDARD')
 412   }
 413
 414   command = await presetVideo(command, options.inputPath, options, fps)
 415
 416   if (options.resolution !== undefined) {
 417     // '?x720' or '720x?' for example
 418     const size = options.isPortraitMode === true
 419       ? `${options.resolution}x?`
 420       : `?x${options.resolution}`
 421
 422     command = command.size(size)
 423   }
 424
 425   // Hard FPS limits
 426   if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
 427   else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
 428
 429   command = command.withFPS(fps)
 430
 431   return command
 432 }
 433
 434 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 435   command = command.loop(undefined)
 436
 437   command = await presetVideo(command, options.audioPath, options)
 438
 439   /*
 440   MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html
 441   Our target situation is closer to a livestream than a stream,
 442   since we want to reduce as much a possible the encoding burden,
 443   although not to the point of a livestream where there is a hard
 444   constraint on the frames per second to be encoded.
 445   */
 446   command.outputOption('-preset:v veryfast')
 447
 448   command = command.input(options.audioPath)
 449                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 450                    .outputOption('-tune stillimage')
 451                    .outputOption('-shortest')
 452
 453   return command
 454 }
 455
 456 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, _options: OnlyAudioTranscodeOptions) {
 457   command = presetOnlyAudio(command)
 458
 459   return command
 460 }
 461
 462 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 463   command = presetCopy(command)
 464
 465   command = command.outputOption('-map_metadata -1') // strip all metadata
 466                    .outputOption('-movflags faststart')
 467
 468   return command
 469 }
 470
 471 async function buildHLSVODCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 472   const videoPath = getHLSVideoPath(options)
 473
 474   if (options.copyCodecs) command = presetCopy(command)
 475   else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
 476   else command = await buildx264VODCommand(command, options)
 477
 478   command = command.outputOption('-hls_time 4')
 479                    .outputOption('-hls_list_size 0')
 480                    .outputOption('-hls_playlist_type vod')
 481                    .outputOption('-hls_segment_filename ' + videoPath)
 482                    .outputOption('-hls_segment_type fmp4')
 483                    .outputOption('-f hls')
 484                    .outputOption('-hls_flags single_file')
 485
 486   return command
 487 }
 488
 489 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 490   if (options.type !== 'hls') return
 491
 492   const fileContent = await readFile(options.outputPath)
 493
 494   const videoFileName = options.hlsPlaylist.videoFilename
 495   const videoFilePath = getHLSVideoPath(options)
 496
 497   // Fix wrong mapping with some ffmpeg versions
 498   const newContent = fileContent.toString()
 499                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 500
 501   await writeFile(options.outputPath, newContent)
 502 }
 503
 504 function getHLSVideoPath (options: HLSTranscodeOptions) {
 505   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 506 }
 507
 508 // ---------------------------------------------------------------------------
 509 // Transcoding presets
 510 // ---------------------------------------------------------------------------
 511
 512 async function getEncoderBuilderResult (options: {
 513   streamType: string
 514   input: string
 515
 516   availableEncoders: AvailableEncoders
 517   profile: string
 518
 519   videoType: 'vod' | 'live'
 520
 521   resolution: number
 522   fps?: number
 523   streamNum?: number
 524 }) {
 525   const { availableEncoders, input, profile, resolution, streamType, fps, streamNum, videoType } = options
 526
 527   const encodersToTry: string[] = VIDEO_TRANSCODING_ENCODERS[streamType]
 528
 529   for (const encoder of encodersToTry) {
 530     if (!(await checkFFmpegEncoders()).get(encoder) || !availableEncoders[videoType][encoder]) continue
 531
 532     const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = availableEncoders[videoType][encoder]
 533     let builder = builderProfiles[profile]
 534
 535     if (!builder) {
 536       logger.debug('Profile %s for encoder %s not available. Fallback to default.', profile, encoder)
 537       builder = builderProfiles.default
 538     }
 539
 540     const result = await builder({ input, resolution: resolution, fps, streamNum })
 541
 542     return {
 543       result,
 544
 545       // If we don't have output options, then copy the input stream
 546       encoder: result.copy === true
 547         ? 'copy'
 548         : encoder
 549     }
 550   }
 551
 552   return null
 553 }
 554
 555 async function presetVideo (
 556   command: ffmpeg.FfmpegCommand,
 557   input: string,
 558   transcodeOptions: TranscodeOptions,
 559   fps?: number
 560 ) {
 561   let localCommand = command
 562     .format('mp4')
 563     .outputOption('-movflags faststart')
 564
 565   // Audio encoder
 566   const parsedAudio = await getAudioStream(input)
 567
 568   let streamsToProcess = [ 'AUDIO', 'VIDEO' ]
 569
 570   if (!parsedAudio.audioStream) {
 571     localCommand = localCommand.noAudio()
 572     streamsToProcess = [ 'VIDEO' ]
 573   }
 574
 575   for (const streamType of streamsToProcess) {
 576     const { profile, resolution, availableEncoders } = transcodeOptions
 577
 578     const builderResult = await getEncoderBuilderResult({
 579       streamType,
 580       input,
 581       resolution,
 582       availableEncoders,
 583       profile,
 584       fps,
 585       videoType: 'vod' as 'vod'
 586     })
 587
 588     if (!builderResult) {
 589       throw new Error('No available encoder found for stream ' + streamType)
 590     }
 591
 592     logger.debug('Apply ffmpeg params from %s.', builderResult.encoder, builderResult)
 593
 594     if (streamType === 'VIDEO') {
 595       localCommand.videoCodec(builderResult.encoder)
 596     } else if (streamType === 'AUDIO') {
 597       localCommand.audioCodec(builderResult.encoder)
 598     }
 599
 600     command.addOutputOptions(builderResult.result.outputOptions)
 601     addDefaultEncoderParams({ command: localCommand, encoder: builderResult.encoder, fps })
 602   }
 603
 604   return localCommand
 605 }
 606
 607 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 608   return command
 609     .format('mp4')
 610     .videoCodec('copy')
 611     .audioCodec('copy')
 612 }
 613
 614 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 615   return command
 616     .format('mp4')
 617     .audioCodec('copy')
 618     .noVideo()
 619 }
 620
 621 // ---------------------------------------------------------------------------
 622 // Utils
 623 // ---------------------------------------------------------------------------
 624
 625 function getFFmpeg (input: string) {
 626   // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems
 627   const command = ffmpeg(input, { niceness: FFMPEG_NICE.TRANSCODING, cwd: CONFIG.STORAGE.TMP_DIR })
 628
 629   if (CONFIG.TRANSCODING.THREADS > 0) {
 630     // If we don't set any threads ffmpeg will chose automatically
 631     command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
 632   }
 633
 634   return command
 635 }
 636
 637 async function runCommand (command: ffmpeg.FfmpegCommand, onEnd?: Function) {
 638   return new Promise<void>((res, rej) => {
 639     command.on('error', (err, stdout, stderr) => {
 640       if (onEnd) onEnd()
 641
 642       logger.error('Error in transcoding job.', { stdout, stderr })
 643       rej(err)
 644     })
 645
 646     command.on('end', () => {
 647       if (onEnd) onEnd()
 648
 649       res()
 650     })
 651
 652     command.run()
 653   })
 654 }