server/helpers/ffmpeg-utils.ts

   1 import { Job } from 'bull'
   2 import * as ffmpeg from 'fluent-ffmpeg'
   3 import { readFile, remove, writeFile } from 'fs-extra'
   4 import { dirname, join } from 'path'
   5 import { FFMPEG_NICE, VIDEO_LIVE } from '@server/initializers/constants'
   6 import { VideoResolution } from '../../shared/models/videos'
   7 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   8 import { CONFIG } from '../initializers/config'
   9 import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils'
  10 import { processImage } from './image-utils'
  11 import { logger } from './logger'
  12
  13 /**
  14  *
  15  * Functions that run transcoding/muxing ffmpeg processes
  16  * Mainly called by lib/video-transcoding.ts and lib/live-manager.ts
  17  *
  18  */
  19
  20 // ---------------------------------------------------------------------------
  21 // Encoder options
  22 // ---------------------------------------------------------------------------
  23
  24 // Options builders
  25
  26 export type EncoderOptionsBuilder = (params: {
  27   input: string
  28   resolution: VideoResolution
  29   fps?: number
  30   streamNum?: number
  31 }) => Promise<EncoderOptions> | EncoderOptions
  32
  33 // Options types
  34
  35 export interface EncoderOptions {
  36   copy?: boolean
  37   outputOptions: string[]
  38 }
  39
  40 // All our encoders
  41
  42 export interface EncoderProfile <T> {
  43   [ profile: string ]: T
  44
  45   default: T
  46 }
  47
  48 export type AvailableEncoders = {
  49   live: {
  50     [ encoder: string ]: EncoderProfile<EncoderOptionsBuilder>
  51   }
  52
  53   vod: {
  54     [ encoder: string ]: EncoderProfile<EncoderOptionsBuilder>
  55   }
  56
  57   encodersToTry: {
  58     video: string[]
  59     audio: string[]
  60   }
  61 }
  62
  63 type StreamType = 'audio' | 'video'
  64
  65 // ---------------------------------------------------------------------------
  66 // Image manipulation
  67 // ---------------------------------------------------------------------------
  68
  69 function convertWebPToJPG (path: string, destination: string): Promise<void> {
  70   const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL })
  71     .output(destination)
  72
  73   return runCommand(command)
  74 }
  75
  76 function processGIF (
  77   path: string,
  78   destination: string,
  79   newSize: { width: number, height: number }
  80 ): Promise<void> {
  81   const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL })
  82     .fps(20)
  83     .size(`${newSize.width}x${newSize.height}`)
  84     .output(destination)
  85
  86   return runCommand(command)
  87 }
  88
  89 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
  90   const pendingImageName = 'pending-' + imageName
  91
  92   const options = {
  93     filename: pendingImageName,
  94     count: 1,
  95     folder
  96   }
  97
  98   const pendingImagePath = join(folder, pendingImageName)
  99
 100   try {
 101     await new Promise<string>((res, rej) => {
 102       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
 103         .on('error', rej)
 104         .on('end', () => res(imageName))
 105         .thumbnail(options)
 106     })
 107
 108     const destination = join(folder, imageName)
 109     await processImage(pendingImagePath, destination, size)
 110   } catch (err) {
 111     logger.error('Cannot generate image from video %s.', fromPath, { err })
 112
 113     try {
 114       await remove(pendingImagePath)
 115     } catch (err) {
 116       logger.debug('Cannot remove pending image path after generation error.', { err })
 117     }
 118   }
 119 }
 120
 121 // ---------------------------------------------------------------------------
 122 // Transcode meta function
 123 // ---------------------------------------------------------------------------
 124
 125 type TranscodeOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 126
 127 interface BaseTranscodeOptions {
 128   type: TranscodeOptionsType
 129
 130   inputPath: string
 131   outputPath: string
 132
 133   availableEncoders: AvailableEncoders
 134   profile: string
 135
 136   resolution: VideoResolution
 137
 138   isPortraitMode?: boolean
 139
 140   job?: Job
 141 }
 142
 143 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 144   type: 'hls'
 145   copyCodecs: boolean
 146   hlsPlaylist: {
 147     videoFilename: string
 148   }
 149 }
 150
 151 interface HLSFromTSTranscodeOptions extends BaseTranscodeOptions {
 152   type: 'hls-from-ts'
 153
 154   isAAC: boolean
 155
 156   hlsPlaylist: {
 157     videoFilename: string
 158   }
 159 }
 160
 161 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 162   type: 'quick-transcode'
 163 }
 164
 165 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 166   type: 'video'
 167 }
 168
 169 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 170   type: 'merge-audio'
 171   audioPath: string
 172 }
 173
 174 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 175   type: 'only-audio'
 176 }
 177
 178 type TranscodeOptions =
 179   HLSTranscodeOptions
 180   | HLSFromTSTranscodeOptions
 181   | VideoTranscodeOptions
 182   | MergeAudioTranscodeOptions
 183   | OnlyAudioTranscodeOptions
 184   | QuickTranscodeOptions
 185
 186 const builders: {
 187   [ type in TranscodeOptionsType ]: (c: ffmpeg.FfmpegCommand, o?: TranscodeOptions) => Promise<ffmpeg.FfmpegCommand> | ffmpeg.FfmpegCommand
 188 } = {
 189   'quick-transcode': buildQuickTranscodeCommand,
 190   'hls': buildHLSVODCommand,
 191   'hls-from-ts': buildHLSVODFromTSCommand,
 192   'merge-audio': buildAudioMergeCommand,
 193   'only-audio': buildOnlyAudioCommand,
 194   'video': buildx264VODCommand
 195 }
 196
 197 async function transcode (options: TranscodeOptions) {
 198   logger.debug('Will run transcode.', { options })
 199
 200   let command = getFFmpeg(options.inputPath, 'vod')
 201     .output(options.outputPath)
 202
 203   command = await builders[options.type](command, options)
 204
 205   await runCommand(command, options.job)
 206
 207   await fixHLSPlaylistIfNeeded(options)
 208 }
 209
 210 // ---------------------------------------------------------------------------
 211 // Live muxing/transcoding functions
 212 // ---------------------------------------------------------------------------
 213
 214 async function getLiveTranscodingCommand (options: {
 215   rtmpUrl: string
 216   outPath: string
 217   resolutions: number[]
 218   fps: number
 219
 220   availableEncoders: AvailableEncoders
 221   profile: string
 222 }) {
 223   const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile } = options
 224   const input = rtmpUrl
 225
 226   const command = getFFmpeg(input, 'live')
 227
 228   const varStreamMap: string[] = []
 229
 230   command.complexFilter([
 231     {
 232       inputs: '[v:0]',
 233       filter: 'split',
 234       options: resolutions.length,
 235       outputs: resolutions.map(r => `vtemp${r}`)
 236     },
 237
 238     ...resolutions.map(r => ({
 239       inputs: `vtemp${r}`,
 240       filter: 'scale',
 241       options: `w=-2:h=${r}`,
 242       outputs: `vout${r}`
 243     }))
 244   ])
 245
 246   command.outputOption('-preset superfast')
 247   command.outputOption('-sc_threshold 0')
 248
 249   addDefaultEncoderGlobalParams({ command })
 250
 251   for (let i = 0; i < resolutions.length; i++) {
 252     const resolution = resolutions[i]
 253     const resolutionFPS = computeFPS(fps, resolution)
 254
 255     const baseEncoderBuilderParams = {
 256       input,
 257
 258       availableEncoders,
 259       profile,
 260
 261       fps: resolutionFPS,
 262       resolution,
 263       streamNum: i,
 264       videoType: 'live' as 'live'
 265     }
 266
 267     {
 268       const streamType: StreamType = 'video'
 269       const builderResult = await getEncoderBuilderResult(Object.assign({}, baseEncoderBuilderParams, { streamType }))
 270       if (!builderResult) {
 271         throw new Error('No available live video encoder found')
 272       }
 273
 274       command.outputOption(`-map [vout${resolution}]`)
 275
 276       addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
 277
 278       logger.debug('Apply ffmpeg live video params from %s.', builderResult.encoder, builderResult)
 279
 280       command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`)
 281       command.addOutputOptions(builderResult.result.outputOptions)
 282     }
 283
 284     {
 285       const streamType: StreamType = 'audio'
 286       const builderResult = await getEncoderBuilderResult(Object.assign({}, baseEncoderBuilderParams, { streamType }))
 287       if (!builderResult) {
 288         throw new Error('No available live audio encoder found')
 289       }
 290
 291       command.outputOption('-map a:0')
 292
 293       addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
 294
 295       logger.debug('Apply ffmpeg live audio params from %s.', builderResult.encoder, builderResult)
 296
 297       command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`)
 298       command.addOutputOptions(builderResult.result.outputOptions)
 299     }
 300
 301     varStreamMap.push(`v:${i},a:${i}`)
 302   }
 303
 304   addDefaultLiveHLSParams(command, outPath)
 305
 306   command.outputOption('-var_stream_map', varStreamMap.join(' '))
 307
 308   return command
 309 }
 310
 311 function getLiveMuxingCommand (rtmpUrl: string, outPath: string) {
 312   const command = getFFmpeg(rtmpUrl, 'live')
 313
 314   command.outputOption('-c:v copy')
 315   command.outputOption('-c:a copy')
 316   command.outputOption('-map 0:a?')
 317   command.outputOption('-map 0:v?')
 318
 319   addDefaultLiveHLSParams(command, outPath)
 320
 321   return command
 322 }
 323
 324 function buildStreamSuffix (base: string, streamNum?: number) {
 325   if (streamNum !== undefined) {
 326     return `${base}:${streamNum}`
 327   }
 328
 329   return base
 330 }
 331
 332 // ---------------------------------------------------------------------------
 333 // Default options
 334 // ---------------------------------------------------------------------------
 335
 336 function addDefaultEncoderGlobalParams (options: {
 337   command: ffmpeg.FfmpegCommand
 338 }) {
 339   const { command } = options
 340
 341   // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375
 342   command.outputOption('-max_muxing_queue_size 1024')
 343          // strip all metadata
 344          .outputOption('-map_metadata -1')
 345          // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 346          .outputOption('-b_strategy 1')
 347          // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 348          .outputOption('-bf 16')
 349          // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 350          .outputOption('-pix_fmt yuv420p')
 351 }
 352
 353 function addDefaultEncoderParams (options: {
 354   command: ffmpeg.FfmpegCommand
 355   encoder: 'libx264' | string
 356   streamNum?: number
 357   fps?: number
 358 }) {
 359   const { command, encoder, fps, streamNum } = options
 360
 361   if (encoder === 'libx264') {
 362     // 3.1 is the minimal resource allocation for our highest supported resolution
 363     command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1')
 364
 365     if (fps) {
 366       // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 367       // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 368       // https://superuser.com/a/908325
 369       command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2))
 370     }
 371   }
 372 }
 373
 374 function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) {
 375   command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS)
 376   command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
 377   command.outputOption('-hls_flags delete_segments+independent_segments')
 378   command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
 379   command.outputOption('-master_pl_name master.m3u8')
 380   command.outputOption(`-f hls`)
 381
 382   command.output(join(outPath, '%v.m3u8'))
 383 }
 384
 385 // ---------------------------------------------------------------------------
 386 // Transcode VOD command builders
 387 // ---------------------------------------------------------------------------
 388
 389 async function buildx264VODCommand (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 390   let fps = await getVideoFileFPS(options.inputPath)
 391   fps = computeFPS(fps, options.resolution)
 392
 393   command = await presetVideo(command, options.inputPath, options, fps)
 394
 395   if (options.resolution !== undefined) {
 396     // '?x720' or '720x?' for example
 397     const size = options.isPortraitMode === true
 398       ? `${options.resolution}x?`
 399       : `?x${options.resolution}`
 400
 401     command = command.size(size)
 402   }
 403
 404   return command
 405 }
 406
 407 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 408   command = command.loop(undefined)
 409
 410   command = await presetVideo(command, options.audioPath, options)
 411
 412   command.outputOption('-preset:v veryfast')
 413
 414   command = command.input(options.audioPath)
 415                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 416                    .outputOption('-tune stillimage')
 417                    .outputOption('-shortest')
 418
 419   return command
 420 }
 421
 422 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, _options: OnlyAudioTranscodeOptions) {
 423   command = presetOnlyAudio(command)
 424
 425   return command
 426 }
 427
 428 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 429   command = presetCopy(command)
 430
 431   command = command.outputOption('-map_metadata -1') // strip all metadata
 432                    .outputOption('-movflags faststart')
 433
 434   return command
 435 }
 436
 437 function addCommonHLSVODCommandOptions (command: ffmpeg.FfmpegCommand, outputPath: string) {
 438   return command.outputOption('-hls_time 4')
 439                 .outputOption('-hls_list_size 0')
 440                 .outputOption('-hls_playlist_type vod')
 441                 .outputOption('-hls_segment_filename ' + outputPath)
 442                 .outputOption('-hls_segment_type fmp4')
 443                 .outputOption('-f hls')
 444                 .outputOption('-hls_flags single_file')
 445 }
 446
 447 async function buildHLSVODCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 448   const videoPath = getHLSVideoPath(options)
 449
 450   if (options.copyCodecs) command = presetCopy(command)
 451   else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
 452   else command = await buildx264VODCommand(command, options)
 453
 454   addCommonHLSVODCommandOptions(command, videoPath)
 455
 456   return command
 457 }
 458
 459 async function buildHLSVODFromTSCommand (command: ffmpeg.FfmpegCommand, options: HLSFromTSTranscodeOptions) {
 460   const videoPath = getHLSVideoPath(options)
 461
 462   command.outputOption('-c copy')
 463
 464   if (options.isAAC) {
 465     // Required for example when copying an AAC stream from an MPEG-TS
 466     // Since it's a bitstream filter, we don't need to reencode the audio
 467     command.outputOption('-bsf:a aac_adtstoasc')
 468   }
 469
 470   addCommonHLSVODCommandOptions(command, videoPath)
 471
 472   return command
 473 }
 474
 475 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 476   if (options.type !== 'hls' && options.type !== 'hls-from-ts') return
 477
 478   const fileContent = await readFile(options.outputPath)
 479
 480   const videoFileName = options.hlsPlaylist.videoFilename
 481   const videoFilePath = getHLSVideoPath(options)
 482
 483   // Fix wrong mapping with some ffmpeg versions
 484   const newContent = fileContent.toString()
 485                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 486
 487   await writeFile(options.outputPath, newContent)
 488 }
 489
 490 function getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) {
 491   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 492 }
 493
 494 // ---------------------------------------------------------------------------
 495 // Transcoding presets
 496 // ---------------------------------------------------------------------------
 497
 498 // Run encoder builder depending on available encoders
 499 // Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one
 500 // If the default one does not exist, check the next encoder
 501 async function getEncoderBuilderResult (options: {
 502   streamType: 'video' | 'audio'
 503   input: string
 504
 505   availableEncoders: AvailableEncoders
 506   profile: string
 507
 508   videoType: 'vod' | 'live'
 509
 510   resolution: number
 511   fps?: number
 512   streamNum?: number
 513 }) {
 514   const { availableEncoders, input, profile, resolution, streamType, fps, streamNum, videoType } = options
 515
 516   const encodersToTry = availableEncoders.encodersToTry[streamType]
 517   const encoders = availableEncoders[videoType]
 518
 519   for (const encoder of encodersToTry) {
 520     if (!(await checkFFmpegEncoders()).get(encoder) || !encoders[encoder]) continue
 521
 522     // An object containing available profiles for this encoder
 523     const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = encoders[encoder]
 524     let builder = builderProfiles[profile]
 525
 526     if (!builder) {
 527       logger.debug('Profile %s for encoder %s not available. Fallback to default.', profile, encoder)
 528       builder = builderProfiles.default
 529
 530       if (!builder) {
 531         logger.debug('Default profile for encoder %s not available. Try next available encoder.', encoder)
 532         continue
 533       }
 534     }
 535
 536     const result = await builder({ input, resolution: resolution, fps, streamNum })
 537
 538     return {
 539       result,
 540
 541       // If we don't have output options, then copy the input stream
 542       encoder: result.copy === true
 543         ? 'copy'
 544         : encoder
 545     }
 546   }
 547
 548   return null
 549 }
 550
 551 async function presetVideo (
 552   command: ffmpeg.FfmpegCommand,
 553   input: string,
 554   transcodeOptions: TranscodeOptions,
 555   fps?: number
 556 ) {
 557   let localCommand = command
 558     .format('mp4')
 559     .outputOption('-movflags faststart')
 560
 561   addDefaultEncoderGlobalParams({ command })
 562
 563   // Audio encoder
 564   const parsedAudio = await getAudioStream(input)
 565
 566   let streamsToProcess: StreamType[] = [ 'audio', 'video' ]
 567
 568   if (!parsedAudio.audioStream) {
 569     localCommand = localCommand.noAudio()
 570     streamsToProcess = [ 'audio' ]
 571   }
 572
 573   for (const streamType of streamsToProcess) {
 574     const { profile, resolution, availableEncoders } = transcodeOptions
 575
 576     const builderResult = await getEncoderBuilderResult({
 577       streamType,
 578       input,
 579       resolution,
 580       availableEncoders,
 581       profile,
 582       fps,
 583       videoType: 'vod' as 'vod'
 584     })
 585
 586     if (!builderResult) {
 587       throw new Error('No available encoder found for stream ' + streamType)
 588     }
 589
 590     logger.debug('Apply ffmpeg params from %s.', builderResult.encoder, builderResult)
 591
 592     if (streamType === 'video') {
 593       localCommand.videoCodec(builderResult.encoder)
 594     } else if (streamType === 'audio') {
 595       localCommand.audioCodec(builderResult.encoder)
 596     }
 597
 598     command.addOutputOptions(builderResult.result.outputOptions)
 599     addDefaultEncoderParams({ command: localCommand, encoder: builderResult.encoder, fps })
 600   }
 601
 602   return localCommand
 603 }
 604
 605 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 606   return command
 607     .format('mp4')
 608     .videoCodec('copy')
 609     .audioCodec('copy')
 610 }
 611
 612 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 613   return command
 614     .format('mp4')
 615     .audioCodec('copy')
 616     .noVideo()
 617 }
 618
 619 // ---------------------------------------------------------------------------
 620 // Utils
 621 // ---------------------------------------------------------------------------
 622
 623 function getFFmpeg (input: string, type: 'live' | 'vod') {
 624   // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems
 625   const command = ffmpeg(input, {
 626     niceness: type === 'live' ? FFMPEG_NICE.LIVE : FFMPEG_NICE.VOD,
 627     cwd: CONFIG.STORAGE.TMP_DIR
 628   })
 629
 630   const threads = type === 'live'
 631     ? CONFIG.LIVE.TRANSCODING.THREADS
 632     : CONFIG.TRANSCODING.THREADS
 633
 634   if (threads > 0) {
 635     // If we don't set any threads ffmpeg will chose automatically
 636     command.outputOption('-threads ' + threads)
 637   }
 638
 639   return command
 640 }
 641
 642 async function runCommand (command: ffmpeg.FfmpegCommand, job?: Job) {
 643   return new Promise<void>((res, rej) => {
 644     command.on('error', (err, stdout, stderr) => {
 645       logger.error('Error in transcoding job.', { stdout, stderr })
 646       rej(err)
 647     })
 648
 649     command.on('end', (stdout, stderr) => {
 650       logger.debug('FFmpeg command ended.', { stdout, stderr })
 651
 652       res()
 653     })
 654
 655     if (job) {
 656       command.on('progress', progress => {
 657         if (!progress.percent) return
 658
 659         job.progress(Math.round(progress.percent))
 660           .catch(err => logger.warn('Cannot set ffmpeg job progress.', { err }))
 661       })
 662     }
 663
 664     command.run()
 665   })
 666 }
 667
 668 // ---------------------------------------------------------------------------
 669
 670 export {
 671   getLiveTranscodingCommand,
 672   getLiveMuxingCommand,
 673   buildStreamSuffix,
 674   convertWebPToJPG,
 675   processGIF,
 676   generateImageFromVideoFile,
 677   TranscodeOptions,
 678   TranscodeOptionsType,
 679   transcode,
 680   runCommand,
 681
 682   // builders
 683   buildx264VODCommand
 684 }