server/helpers/ffmpeg-utils.ts

   1 import { Job } from 'bull'
   2 import * as ffmpeg from 'fluent-ffmpeg'
   3 import { readFile, remove, writeFile } from 'fs-extra'
   4 import { dirname, join } from 'path'
   5 import { FFMPEG_NICE, VIDEO_LIVE, VIDEO_TRANSCODING_ENCODERS } from '@server/initializers/constants'
   6 import { VideoResolution } from '../../shared/models/videos'
   7 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   8 import { CONFIG } from '../initializers/config'
   9 import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils'
  10 import { processImage } from './image-utils'
  11 import { logger } from './logger'
  12
  13 /**
  14  *
  15  * Functions that run transcoding/muxing ffmpeg processes
  16  * Mainly called by lib/video-transcoding.ts and lib/live-manager.ts
  17  *
  18  */
  19
  20 // ---------------------------------------------------------------------------
  21 // Encoder options
  22 // ---------------------------------------------------------------------------
  23
  24 // Options builders
  25
  26 export type EncoderOptionsBuilder = (params: {
  27   input: string
  28   resolution: VideoResolution
  29   fps?: number
  30   streamNum?: number
  31 }) => Promise<EncoderOptions> | EncoderOptions
  32
  33 // Options types
  34
  35 export interface EncoderOptions {
  36   copy?: boolean
  37   outputOptions: string[]
  38 }
  39
  40 // All our encoders
  41
  42 export interface EncoderProfile <T> {
  43   [ profile: string ]: T
  44
  45   default: T
  46 }
  47
  48 export type AvailableEncoders = {
  49   [ id in 'live' | 'vod' ]: {
  50     [ encoder in 'libx264' | 'aac' | 'libfdk_aac' ]?: EncoderProfile<EncoderOptionsBuilder>
  51   }
  52 }
  53
  54 // ---------------------------------------------------------------------------
  55 // Image manipulation
  56 // ---------------------------------------------------------------------------
  57
  58 function convertWebPToJPG (path: string, destination: string): Promise<void> {
  59   const command = ffmpeg(path)
  60     .output(destination)
  61
  62   return runCommand(command)
  63 }
  64
  65 function processGIF (
  66   path: string,
  67   destination: string,
  68   newSize: { width: number, height: number }
  69 ): Promise<void> {
  70   const command = ffmpeg(path)
  71     .fps(20)
  72     .size(`${newSize.width}x${newSize.height}`)
  73     .output(destination)
  74
  75   return runCommand(command)
  76 }
  77
  78 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
  79   const pendingImageName = 'pending-' + imageName
  80
  81   const options = {
  82     filename: pendingImageName,
  83     count: 1,
  84     folder
  85   }
  86
  87   const pendingImagePath = join(folder, pendingImageName)
  88
  89   try {
  90     await new Promise<string>((res, rej) => {
  91       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
  92         .on('error', rej)
  93         .on('end', () => res(imageName))
  94         .thumbnail(options)
  95     })
  96
  97     const destination = join(folder, imageName)
  98     await processImage(pendingImagePath, destination, size)
  99   } catch (err) {
 100     logger.error('Cannot generate image from video %s.', fromPath, { err })
 101
 102     try {
 103       await remove(pendingImagePath)
 104     } catch (err) {
 105       logger.debug('Cannot remove pending image path after generation error.', { err })
 106     }
 107   }
 108 }
 109
 110 // ---------------------------------------------------------------------------
 111 // Transcode meta function
 112 // ---------------------------------------------------------------------------
 113
 114 type TranscodeOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 115
 116 interface BaseTranscodeOptions {
 117   type: TranscodeOptionsType
 118
 119   inputPath: string
 120   outputPath: string
 121
 122   availableEncoders: AvailableEncoders
 123   profile: string
 124
 125   resolution: VideoResolution
 126
 127   isPortraitMode?: boolean
 128
 129   job?: Job
 130 }
 131
 132 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 133   type: 'hls'
 134   copyCodecs: boolean
 135   hlsPlaylist: {
 136     videoFilename: string
 137   }
 138 }
 139
 140 interface HLSFromTSTranscodeOptions extends BaseTranscodeOptions {
 141   type: 'hls-from-ts'
 142
 143   isAAC: boolean
 144
 145   hlsPlaylist: {
 146     videoFilename: string
 147   }
 148 }
 149
 150 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 151   type: 'quick-transcode'
 152 }
 153
 154 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 155   type: 'video'
 156 }
 157
 158 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 159   type: 'merge-audio'
 160   audioPath: string
 161 }
 162
 163 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 164   type: 'only-audio'
 165 }
 166
 167 type TranscodeOptions =
 168   HLSTranscodeOptions
 169   | HLSFromTSTranscodeOptions
 170   | VideoTranscodeOptions
 171   | MergeAudioTranscodeOptions
 172   | OnlyAudioTranscodeOptions
 173   | QuickTranscodeOptions
 174
 175 const builders: {
 176   [ type in TranscodeOptionsType ]: (c: ffmpeg.FfmpegCommand, o?: TranscodeOptions) => Promise<ffmpeg.FfmpegCommand> | ffmpeg.FfmpegCommand
 177 } = {
 178   'quick-transcode': buildQuickTranscodeCommand,
 179   'hls': buildHLSVODCommand,
 180   'hls-from-ts': buildHLSVODFromTSCommand,
 181   'merge-audio': buildAudioMergeCommand,
 182   'only-audio': buildOnlyAudioCommand,
 183   'video': buildx264VODCommand
 184 }
 185
 186 async function transcode (options: TranscodeOptions) {
 187   logger.debug('Will run transcode.', { options })
 188
 189   let command = getFFmpeg(options.inputPath, 'vod')
 190     .output(options.outputPath)
 191
 192   command = await builders[options.type](command, options)
 193
 194   await runCommand(command, options.job)
 195
 196   await fixHLSPlaylistIfNeeded(options)
 197 }
 198
 199 // ---------------------------------------------------------------------------
 200 // Live muxing/transcoding functions
 201 // ---------------------------------------------------------------------------
 202
 203 async function getLiveTranscodingCommand (options: {
 204   rtmpUrl: string
 205   outPath: string
 206   resolutions: number[]
 207   fps: number
 208
 209   availableEncoders: AvailableEncoders
 210   profile: string
 211 }) {
 212   const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile } = options
 213   const input = rtmpUrl
 214
 215   const command = getFFmpeg(input, 'live')
 216
 217   const varStreamMap: string[] = []
 218
 219   command.complexFilter([
 220     {
 221       inputs: '[v:0]',
 222       filter: 'split',
 223       options: resolutions.length,
 224       outputs: resolutions.map(r => `vtemp${r}`)
 225     },
 226
 227     ...resolutions.map(r => ({
 228       inputs: `vtemp${r}`,
 229       filter: 'scale',
 230       options: `w=-2:h=${r}`,
 231       outputs: `vout${r}`
 232     }))
 233   ])
 234
 235   command.outputOption('-preset superfast')
 236   command.outputOption('-sc_threshold 0')
 237
 238   addDefaultEncoderGlobalParams({ command })
 239
 240   for (let i = 0; i < resolutions.length; i++) {
 241     const resolution = resolutions[i]
 242     const resolutionFPS = computeFPS(fps, resolution)
 243
 244     const baseEncoderBuilderParams = {
 245       input,
 246       availableEncoders,
 247       profile,
 248       fps: resolutionFPS,
 249       resolution,
 250       streamNum: i,
 251       videoType: 'live' as 'live'
 252     }
 253
 254     {
 255       const builderResult = await getEncoderBuilderResult(Object.assign({}, baseEncoderBuilderParams, { streamType: 'VIDEO' }))
 256       if (!builderResult) {
 257         throw new Error('No available live video encoder found')
 258       }
 259
 260       command.outputOption(`-map [vout${resolution}]`)
 261
 262       addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
 263
 264       logger.debug('Apply ffmpeg live video params from %s.', builderResult.encoder, builderResult)
 265
 266       command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`)
 267       command.addOutputOptions(builderResult.result.outputOptions)
 268     }
 269
 270     {
 271       const builderResult = await getEncoderBuilderResult(Object.assign({}, baseEncoderBuilderParams, { streamType: 'AUDIO' }))
 272       if (!builderResult) {
 273         throw new Error('No available live audio encoder found')
 274       }
 275
 276       command.outputOption('-map a:0')
 277
 278       addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
 279
 280       logger.debug('Apply ffmpeg live audio params from %s.', builderResult.encoder, builderResult)
 281
 282       command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`)
 283       command.addOutputOptions(builderResult.result.outputOptions)
 284     }
 285
 286     varStreamMap.push(`v:${i},a:${i}`)
 287   }
 288
 289   addDefaultLiveHLSParams(command, outPath)
 290
 291   command.outputOption('-var_stream_map', varStreamMap.join(' '))
 292
 293   return command
 294 }
 295
 296 function getLiveMuxingCommand (rtmpUrl: string, outPath: string) {
 297   const command = getFFmpeg(rtmpUrl, 'live')
 298
 299   command.outputOption('-c:v copy')
 300   command.outputOption('-c:a copy')
 301   command.outputOption('-map 0:a?')
 302   command.outputOption('-map 0:v?')
 303
 304   addDefaultLiveHLSParams(command, outPath)
 305
 306   return command
 307 }
 308
 309 function buildStreamSuffix (base: string, streamNum?: number) {
 310   if (streamNum !== undefined) {
 311     return `${base}:${streamNum}`
 312   }
 313
 314   return base
 315 }
 316
 317 // ---------------------------------------------------------------------------
 318 // Default options
 319 // ---------------------------------------------------------------------------
 320
 321 function addDefaultEncoderGlobalParams (options: {
 322   command: ffmpeg.FfmpegCommand
 323 }) {
 324   const { command } = options
 325
 326   // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375
 327   command.outputOption('-max_muxing_queue_size 1024')
 328          // strip all metadata
 329          .outputOption('-map_metadata -1')
 330          // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 331          .outputOption('-b_strategy 1')
 332          // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 333          .outputOption('-bf 16')
 334          // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 335          .outputOption('-pix_fmt yuv420p')
 336 }
 337
 338 function addDefaultEncoderParams (options: {
 339   command: ffmpeg.FfmpegCommand
 340   encoder: 'libx264' | string
 341   streamNum?: number
 342   fps?: number
 343 }) {
 344   const { command, encoder, fps, streamNum } = options
 345
 346   if (encoder === 'libx264') {
 347     // 3.1 is the minimal resource allocation for our highest supported resolution
 348     command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1')
 349
 350     if (fps) {
 351       // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 352       // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 353       // https://superuser.com/a/908325
 354       command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2))
 355     }
 356   }
 357 }
 358
 359 function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) {
 360   command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS)
 361   command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
 362   command.outputOption('-hls_flags delete_segments+independent_segments')
 363   command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
 364   command.outputOption('-master_pl_name master.m3u8')
 365   command.outputOption(`-f hls`)
 366
 367   command.output(join(outPath, '%v.m3u8'))
 368 }
 369
 370 // ---------------------------------------------------------------------------
 371 // Transcode VOD command builders
 372 // ---------------------------------------------------------------------------
 373
 374 async function buildx264VODCommand (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 375   let fps = await getVideoFileFPS(options.inputPath)
 376   fps = computeFPS(fps, options.resolution)
 377
 378   command = await presetVideo(command, options.inputPath, options, fps)
 379
 380   if (options.resolution !== undefined) {
 381     // '?x720' or '720x?' for example
 382     const size = options.isPortraitMode === true
 383       ? `${options.resolution}x?`
 384       : `?x${options.resolution}`
 385
 386     command = command.size(size)
 387   }
 388
 389   return command
 390 }
 391
 392 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 393   command = command.loop(undefined)
 394
 395   command = await presetVideo(command, options.audioPath, options)
 396
 397   command.outputOption('-preset:v veryfast')
 398
 399   command = command.input(options.audioPath)
 400                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 401                    .outputOption('-tune stillimage')
 402                    .outputOption('-shortest')
 403
 404   return command
 405 }
 406
 407 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, _options: OnlyAudioTranscodeOptions) {
 408   command = presetOnlyAudio(command)
 409
 410   return command
 411 }
 412
 413 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 414   command = presetCopy(command)
 415
 416   command = command.outputOption('-map_metadata -1') // strip all metadata
 417                    .outputOption('-movflags faststart')
 418
 419   return command
 420 }
 421
 422 function addCommonHLSVODCommandOptions (command: ffmpeg.FfmpegCommand, outputPath: string) {
 423   return command.outputOption('-hls_time 4')
 424                 .outputOption('-hls_list_size 0')
 425                 .outputOption('-hls_playlist_type vod')
 426                 .outputOption('-hls_segment_filename ' + outputPath)
 427                 .outputOption('-hls_segment_type fmp4')
 428                 .outputOption('-f hls')
 429                 .outputOption('-hls_flags single_file')
 430 }
 431
 432 async function buildHLSVODCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 433   const videoPath = getHLSVideoPath(options)
 434
 435   if (options.copyCodecs) command = presetCopy(command)
 436   else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
 437   else command = await buildx264VODCommand(command, options)
 438
 439   addCommonHLSVODCommandOptions(command, videoPath)
 440
 441   return command
 442 }
 443
 444 async function buildHLSVODFromTSCommand (command: ffmpeg.FfmpegCommand, options: HLSFromTSTranscodeOptions) {
 445   const videoPath = getHLSVideoPath(options)
 446
 447   command.outputOption('-c copy')
 448
 449   if (options.isAAC) {
 450     // Required for example when copying an AAC stream from an MPEG-TS
 451     // Since it's a bitstream filter, we don't need to reencode the audio
 452     command.outputOption('-bsf:a aac_adtstoasc')
 453   }
 454
 455   addCommonHLSVODCommandOptions(command, videoPath)
 456
 457   return command
 458 }
 459
 460 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 461   if (options.type !== 'hls' && options.type !== 'hls-from-ts') return
 462
 463   const fileContent = await readFile(options.outputPath)
 464
 465   const videoFileName = options.hlsPlaylist.videoFilename
 466   const videoFilePath = getHLSVideoPath(options)
 467
 468   // Fix wrong mapping with some ffmpeg versions
 469   const newContent = fileContent.toString()
 470                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 471
 472   await writeFile(options.outputPath, newContent)
 473 }
 474
 475 function getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) {
 476   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 477 }
 478
 479 // ---------------------------------------------------------------------------
 480 // Transcoding presets
 481 // ---------------------------------------------------------------------------
 482
 483 async function getEncoderBuilderResult (options: {
 484   streamType: string
 485   input: string
 486
 487   availableEncoders: AvailableEncoders
 488   profile: string
 489
 490   videoType: 'vod' | 'live'
 491
 492   resolution: number
 493   fps?: number
 494   streamNum?: number
 495 }) {
 496   const { availableEncoders, input, profile, resolution, streamType, fps, streamNum, videoType } = options
 497
 498   const encodersToTry: string[] = VIDEO_TRANSCODING_ENCODERS[streamType]
 499
 500   for (const encoder of encodersToTry) {
 501     if (!(await checkFFmpegEncoders()).get(encoder) || !availableEncoders[videoType][encoder]) continue
 502
 503     const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = availableEncoders[videoType][encoder]
 504     let builder = builderProfiles[profile]
 505
 506     if (!builder) {
 507       logger.debug('Profile %s for encoder %s not available. Fallback to default.', profile, encoder)
 508       builder = builderProfiles.default
 509     }
 510
 511     const result = await builder({ input, resolution: resolution, fps, streamNum })
 512
 513     return {
 514       result,
 515
 516       // If we don't have output options, then copy the input stream
 517       encoder: result.copy === true
 518         ? 'copy'
 519         : encoder
 520     }
 521   }
 522
 523   return null
 524 }
 525
 526 async function presetVideo (
 527   command: ffmpeg.FfmpegCommand,
 528   input: string,
 529   transcodeOptions: TranscodeOptions,
 530   fps?: number
 531 ) {
 532   let localCommand = command
 533     .format('mp4')
 534     .outputOption('-movflags faststart')
 535
 536   addDefaultEncoderGlobalParams({ command })
 537
 538   // Audio encoder
 539   const parsedAudio = await getAudioStream(input)
 540
 541   let streamsToProcess = [ 'AUDIO', 'VIDEO' ]
 542
 543   if (!parsedAudio.audioStream) {
 544     localCommand = localCommand.noAudio()
 545     streamsToProcess = [ 'VIDEO' ]
 546   }
 547
 548   for (const streamType of streamsToProcess) {
 549     const { profile, resolution, availableEncoders } = transcodeOptions
 550
 551     const builderResult = await getEncoderBuilderResult({
 552       streamType,
 553       input,
 554       resolution,
 555       availableEncoders,
 556       profile,
 557       fps,
 558       videoType: 'vod' as 'vod'
 559     })
 560
 561     if (!builderResult) {
 562       throw new Error('No available encoder found for stream ' + streamType)
 563     }
 564
 565     logger.debug('Apply ffmpeg params from %s.', builderResult.encoder, builderResult)
 566
 567     if (streamType === 'VIDEO') {
 568       localCommand.videoCodec(builderResult.encoder)
 569     } else if (streamType === 'AUDIO') {
 570       localCommand.audioCodec(builderResult.encoder)
 571     }
 572
 573     command.addOutputOptions(builderResult.result.outputOptions)
 574     addDefaultEncoderParams({ command: localCommand, encoder: builderResult.encoder, fps })
 575   }
 576
 577   return localCommand
 578 }
 579
 580 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 581   return command
 582     .format('mp4')
 583     .videoCodec('copy')
 584     .audioCodec('copy')
 585 }
 586
 587 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 588   return command
 589     .format('mp4')
 590     .audioCodec('copy')
 591     .noVideo()
 592 }
 593
 594 // ---------------------------------------------------------------------------
 595 // Utils
 596 // ---------------------------------------------------------------------------
 597
 598 function getFFmpeg (input: string, type: 'live' | 'vod') {
 599   // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems
 600   const command = ffmpeg(input, {
 601     niceness: type === 'live' ? FFMPEG_NICE.LIVE : FFMPEG_NICE.VOD,
 602     cwd: CONFIG.STORAGE.TMP_DIR
 603   })
 604
 605   const threads = type === 'live'
 606     ? CONFIG.LIVE.TRANSCODING.THREADS
 607     : CONFIG.TRANSCODING.THREADS
 608
 609   if (threads > 0) {
 610     // If we don't set any threads ffmpeg will chose automatically
 611     command.outputOption('-threads ' + threads)
 612   }
 613
 614   return command
 615 }
 616
 617 async function runCommand (command: ffmpeg.FfmpegCommand, job?: Job) {
 618   return new Promise<void>((res, rej) => {
 619     command.on('error', (err, stdout, stderr) => {
 620       logger.error('Error in transcoding job.', { stdout, stderr })
 621       rej(err)
 622     })
 623
 624     command.on('end', (stdout, stderr) => {
 625       logger.debug('FFmpeg command ended.', { stdout, stderr })
 626
 627       res()
 628     })
 629
 630     if (job) {
 631       command.on('progress', progress => {
 632         if (!progress.percent) return
 633
 634         job.progress(Math.round(progress.percent))
 635           .catch(err => logger.warn('Cannot set ffmpeg job progress.', { err }))
 636       })
 637     }
 638
 639     command.run()
 640   })
 641 }
 642
 643 // ---------------------------------------------------------------------------
 644
 645 export {
 646   getLiveTranscodingCommand,
 647   getLiveMuxingCommand,
 648   buildStreamSuffix,
 649   convertWebPToJPG,
 650   processGIF,
 651   generateImageFromVideoFile,
 652   TranscodeOptions,
 653   TranscodeOptionsType,
 654   transcode,
 655   runCommand,
 656
 657   // builders
 658   buildx264VODCommand
 659 }