server/helpers/ffmpeg-utils.ts

   1 import * as ffmpeg from 'fluent-ffmpeg'
   2 import { readFile, remove, writeFile } from 'fs-extra'
   3 import { dirname, join } from 'path'
   4 import { getTargetBitrate, VideoResolution } from '../../shared/models/videos'
   5 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   6 import { CONFIG } from '../initializers/config'
   7 import { FFMPEG_NICE, VIDEO_LIVE, VIDEO_TRANSCODING_ENCODERS, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
   8 import { getAudioStream, getClosestFramerateStandard, getVideoFileFPS } from './ffprobe-utils'
   9 import { processImage } from './image-utils'
  10 import { logger } from './logger'
  11
  12 // ---------------------------------------------------------------------------
  13 // Encoder options
  14 // ---------------------------------------------------------------------------
  15
  16 // Options builders
  17
  18 export type EncoderOptionsBuilder = (params: {
  19   input: string
  20   resolution: VideoResolution
  21   fps?: number
  22 }) => Promise<EncoderOptions> | EncoderOptions
  23
  24 // Options types
  25
  26 export interface EncoderOptions {
  27   outputOptions: string[]
  28 }
  29
  30 // All our encoders
  31
  32 export interface EncoderProfile <T> {
  33   [ profile: string ]: T
  34
  35   default: T
  36 }
  37
  38 export type AvailableEncoders = {
  39   [ id in 'live' | 'vod' ]: {
  40     [ encoder in 'libx264' | 'aac' | 'libfdkAAC' ]: EncoderProfile<EncoderOptionsBuilder>
  41   }
  42 }
  43
  44 // ---------------------------------------------------------------------------
  45 // Image manipulation
  46 // ---------------------------------------------------------------------------
  47
  48 function convertWebPToJPG (path: string, destination: string): Promise<void> {
  49   const command = ffmpeg(path)
  50     .output(destination)
  51
  52   return runCommand(command)
  53 }
  54
  55 function processGIF (
  56   path: string,
  57   destination: string,
  58   newSize: { width: number, height: number },
  59   keepOriginal = false
  60 ): Promise<void> {
  61   return new Promise<void>(async (res, rej) => {
  62     if (path === destination) {
  63       throw new Error('FFmpeg needs an input path different that the output path.')
  64     }
  65
  66     logger.debug('Processing gif %s to %s.', path, destination)
  67
  68     try {
  69       const command = ffmpeg(path)
  70         .fps(20)
  71         .size(`${newSize.width}x${newSize.height}`)
  72         .output(destination)
  73
  74       command.on('error', (err, stdout, stderr) => {
  75         logger.error('Error in ffmpeg gif resizing process.', { stdout, stderr })
  76         return rej(err)
  77       })
  78       .on('end', async () => {
  79         if (keepOriginal !== true) await remove(path)
  80         res()
  81       })
  82       .run()
  83     } catch (err) {
  84       return rej(err)
  85     }
  86   })
  87 }
  88
  89 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
  90   const pendingImageName = 'pending-' + imageName
  91
  92   const options = {
  93     filename: pendingImageName,
  94     count: 1,
  95     folder
  96   }
  97
  98   const pendingImagePath = join(folder, pendingImageName)
  99
 100   try {
 101     await new Promise<string>((res, rej) => {
 102       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
 103         .on('error', rej)
 104         .on('end', () => res(imageName))
 105         .thumbnail(options)
 106     })
 107
 108     const destination = join(folder, imageName)
 109     await processImage(pendingImagePath, destination, size)
 110   } catch (err) {
 111     logger.error('Cannot generate image from video %s.', fromPath, { err })
 112
 113     try {
 114       await remove(pendingImagePath)
 115     } catch (err) {
 116       logger.debug('Cannot remove pending image path after generation error.', { err })
 117     }
 118   }
 119 }
 120
 121 // ---------------------------------------------------------------------------
 122 // Transcode meta function
 123 // ---------------------------------------------------------------------------
 124
 125 type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 126
 127 interface BaseTranscodeOptions {
 128   type: TranscodeOptionsType
 129
 130   inputPath: string
 131   outputPath: string
 132
 133   availableEncoders: AvailableEncoders
 134   profile: string
 135
 136   resolution: VideoResolution
 137
 138   isPortraitMode?: boolean
 139 }
 140
 141 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 142   type: 'hls'
 143   copyCodecs: boolean
 144   hlsPlaylist: {
 145     videoFilename: string
 146   }
 147 }
 148
 149 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 150   type: 'quick-transcode'
 151 }
 152
 153 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 154   type: 'video'
 155 }
 156
 157 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 158   type: 'merge-audio'
 159   audioPath: string
 160 }
 161
 162 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 163   type: 'only-audio'
 164 }
 165
 166 type TranscodeOptions =
 167   HLSTranscodeOptions
 168   | VideoTranscodeOptions
 169   | MergeAudioTranscodeOptions
 170   | OnlyAudioTranscodeOptions
 171   | QuickTranscodeOptions
 172
 173 const builders: {
 174   [ type in TranscodeOptionsType ]: (c: ffmpeg.FfmpegCommand, o?: TranscodeOptions) => Promise<ffmpeg.FfmpegCommand> | ffmpeg.FfmpegCommand
 175 } = {
 176   'quick-transcode': buildQuickTranscodeCommand,
 177   'hls': buildHLSVODCommand,
 178   'merge-audio': buildAudioMergeCommand,
 179   'only-audio': buildOnlyAudioCommand,
 180   'video': buildx264VODCommand
 181 }
 182
 183 async function transcode (options: TranscodeOptions) {
 184   logger.debug('Will run transcode.', { options })
 185
 186   let command = getFFmpeg(options.inputPath)
 187     .output(options.outputPath)
 188
 189   command = await builders[options.type](command, options)
 190
 191   await runCommand(command)
 192
 193   await fixHLSPlaylistIfNeeded(options)
 194 }
 195
 196 // ---------------------------------------------------------------------------
 197 // Live muxing/transcoding functions
 198 // ---------------------------------------------------------------------------
 199
 200 function getLiveTranscodingCommand (rtmpUrl: string, outPath: string, resolutions: number[], fps: number, deleteSegments: boolean) {
 201   const command = getFFmpeg(rtmpUrl)
 202   command.inputOption('-fflags nobuffer')
 203
 204   const varStreamMap: string[] = []
 205
 206   command.complexFilter([
 207     {
 208       inputs: '[v:0]',
 209       filter: 'split',
 210       options: resolutions.length,
 211       outputs: resolutions.map(r => `vtemp${r}`)
 212     },
 213
 214     ...resolutions.map(r => ({
 215       inputs: `vtemp${r}`,
 216       filter: 'scale',
 217       options: `w=-2:h=${r}`,
 218       outputs: `vout${r}`
 219     }))
 220   ])
 221
 222   addEncoderDefaultParams(command, 'libx264', fps)
 223
 224   command.outputOption('-preset superfast')
 225
 226   for (let i = 0; i < resolutions.length; i++) {
 227     const resolution = resolutions[i]
 228
 229     command.outputOption(`-map [vout${resolution}]`)
 230     command.outputOption(`-c:v:${i} libx264`)
 231     command.outputOption(`-b:v:${i} ${getTargetBitrate(resolution, fps, VIDEO_TRANSCODING_FPS)}`)
 232
 233     command.outputOption(`-map a:0`)
 234     command.outputOption(`-c:a:${i} aac`)
 235
 236     varStreamMap.push(`v:${i},a:${i}`)
 237   }
 238
 239   addDefaultLiveHLSParams(command, outPath, deleteSegments)
 240
 241   command.outputOption('-var_stream_map', varStreamMap.join(' '))
 242
 243   return command
 244 }
 245
 246 function getLiveMuxingCommand (rtmpUrl: string, outPath: string, deleteSegments: boolean) {
 247   const command = getFFmpeg(rtmpUrl)
 248   command.inputOption('-fflags nobuffer')
 249
 250   command.outputOption('-c:v copy')
 251   command.outputOption('-c:a copy')
 252   command.outputOption('-map 0:a?')
 253   command.outputOption('-map 0:v?')
 254
 255   addDefaultLiveHLSParams(command, outPath, deleteSegments)
 256
 257   return command
 258 }
 259
 260 async function hlsPlaylistToFragmentedMP4 (hlsDirectory: string, segmentFiles: string[], outputPath: string) {
 261   const concatFilePath = join(hlsDirectory, 'concat.txt')
 262
 263   function cleaner () {
 264     remove(concatFilePath)
 265       .catch(err => logger.error('Cannot remove concat file in %s.', hlsDirectory, { err }))
 266   }
 267
 268   // First concat the ts files to a mp4 file
 269   const content = segmentFiles.map(f => 'file ' + f)
 270                               .join('\n')
 271
 272   await writeFile(concatFilePath, content + '\n')
 273
 274   const command = getFFmpeg(concatFilePath)
 275   command.inputOption('-safe 0')
 276   command.inputOption('-f concat')
 277
 278   command.outputOption('-c:v copy')
 279   command.audioFilter('aresample=async=1:first_pts=0')
 280   command.output(outputPath)
 281
 282   return runCommand(command, cleaner)
 283 }
 284
 285 // ---------------------------------------------------------------------------
 286
 287 export {
 288   getLiveTranscodingCommand,
 289   getLiveMuxingCommand,
 290   convertWebPToJPG,
 291   processGIF,
 292   generateImageFromVideoFile,
 293   TranscodeOptions,
 294   TranscodeOptionsType,
 295   transcode,
 296   hlsPlaylistToFragmentedMP4
 297 }
 298
 299 // ---------------------------------------------------------------------------
 300
 301 // ---------------------------------------------------------------------------
 302 // Default options
 303 // ---------------------------------------------------------------------------
 304
 305 function addEncoderDefaultParams (command: ffmpeg.FfmpegCommand, encoder: 'libx264' | string, fps?: number) {
 306   if (encoder !== 'libx264') return
 307
 308   command.outputOption('-level 3.1') // 3.1 is the minimal resource allocation for our highest supported resolution
 309          .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 310          .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 311          .outputOption('-pix_fmt yuv420p') // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 312          .outputOption('-map_metadata -1') // strip all metadata
 313          .outputOption('-max_muxing_queue_size 1024') // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375
 314          // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 315          // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 316          // https://superuser.com/a/908325
 317          .outputOption('-g ' + (fps * 2))
 318 }
 319
 320 function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, deleteSegments: boolean) {
 321   command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS)
 322   command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
 323
 324   if (deleteSegments === true) {
 325     command.outputOption('-hls_flags delete_segments')
 326   }
 327
 328   command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
 329   command.outputOption('-master_pl_name master.m3u8')
 330   command.outputOption(`-f hls`)
 331
 332   command.output(join(outPath, '%v.m3u8'))
 333 }
 334
 335 // ---------------------------------------------------------------------------
 336 // Transcode VOD command builders
 337 // ---------------------------------------------------------------------------
 338
 339 async function buildx264VODCommand (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 340   let fps = await getVideoFileFPS(options.inputPath)
 341   if (
 342     // On small/medium resolutions, limit FPS
 343     options.resolution !== undefined &&
 344     options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
 345     fps > VIDEO_TRANSCODING_FPS.AVERAGE
 346   ) {
 347     // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
 348     fps = getClosestFramerateStandard(fps, 'STANDARD')
 349   }
 350
 351   command = await presetVideo(command, options.inputPath, options, fps)
 352
 353   if (options.resolution !== undefined) {
 354     // '?x720' or '720x?' for example
 355     const size = options.isPortraitMode === true ? `${options.resolution}x?` : `?x${options.resolution}`
 356     command = command.size(size)
 357   }
 358
 359   if (fps) {
 360     // Hard FPS limits
 361     if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
 362     else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
 363
 364     command = command.withFPS(fps)
 365   }
 366
 367   return command
 368 }
 369
 370 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 371   command = command.loop(undefined)
 372
 373   command = await presetVideo(command, options.audioPath, options)
 374
 375   /*
 376   MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html
 377   Our target situation is closer to a livestream than a stream,
 378   since we want to reduce as much a possible the encoding burden,
 379   although not to the point of a livestream where there is a hard
 380   constraint on the frames per second to be encoded.
 381   */
 382   command.outputOption('-preset:v veryfast')
 383
 384   command = command.input(options.audioPath)
 385                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 386                    .outputOption('-tune stillimage')
 387                    .outputOption('-shortest')
 388
 389   return command
 390 }
 391
 392 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, _options: OnlyAudioTranscodeOptions) {
 393   command = presetOnlyAudio(command)
 394
 395   return command
 396 }
 397
 398 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 399   command = presetCopy(command)
 400
 401   command = command.outputOption('-map_metadata -1') // strip all metadata
 402                    .outputOption('-movflags faststart')
 403
 404   return command
 405 }
 406
 407 async function buildHLSVODCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 408   const videoPath = getHLSVideoPath(options)
 409
 410   if (options.copyCodecs) command = presetCopy(command)
 411   else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
 412   else command = await buildx264VODCommand(command, options)
 413
 414   command = command.outputOption('-hls_time 4')
 415                    .outputOption('-hls_list_size 0')
 416                    .outputOption('-hls_playlist_type vod')
 417                    .outputOption('-hls_segment_filename ' + videoPath)
 418                    .outputOption('-hls_segment_type fmp4')
 419                    .outputOption('-f hls')
 420                    .outputOption('-hls_flags single_file')
 421
 422   return command
 423 }
 424
 425 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 426   if (options.type !== 'hls') return
 427
 428   const fileContent = await readFile(options.outputPath)
 429
 430   const videoFileName = options.hlsPlaylist.videoFilename
 431   const videoFilePath = getHLSVideoPath(options)
 432
 433   // Fix wrong mapping with some ffmpeg versions
 434   const newContent = fileContent.toString()
 435                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 436
 437   await writeFile(options.outputPath, newContent)
 438 }
 439
 440 function getHLSVideoPath (options: HLSTranscodeOptions) {
 441   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 442 }
 443
 444 // ---------------------------------------------------------------------------
 445 // Transcoding presets
 446 // ---------------------------------------------------------------------------
 447
 448 async function presetVideo (
 449   command: ffmpeg.FfmpegCommand,
 450   input: string,
 451   transcodeOptions: TranscodeOptions,
 452   fps?: number
 453 ) {
 454   let localCommand = command
 455     .format('mp4')
 456     .outputOption('-movflags faststart')
 457
 458   // Audio encoder
 459   const parsedAudio = await getAudioStream(input)
 460
 461   let streamsToProcess = [ 'AUDIO', 'VIDEO' ]
 462   const streamsFound = {
 463     AUDIO: '',
 464     VIDEO: ''
 465   }
 466
 467   if (!parsedAudio.audioStream) {
 468     localCommand = localCommand.noAudio()
 469     streamsToProcess = [ 'VIDEO' ]
 470   }
 471
 472   for (const stream of streamsToProcess) {
 473     const encodersToTry: string[] = VIDEO_TRANSCODING_ENCODERS[stream]
 474
 475     for (const encoder of encodersToTry) {
 476       if (!(await checkFFmpegEncoders()).get(encoder)) continue
 477
 478       const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = transcodeOptions.availableEncoders.vod[encoder]
 479       let builder = builderProfiles[transcodeOptions.profile]
 480
 481       if (!builder) {
 482         logger.debug('Profile %s for encoder %s not available. Fallback to default.', transcodeOptions.profile, encoder)
 483         builder = builderProfiles.default
 484       }
 485
 486       const builderResult = await builder({ input, resolution: transcodeOptions.resolution, fps })
 487
 488       logger.debug('Apply ffmpeg params from %s.', encoder, builderResult)
 489
 490       localCommand.outputOptions(builderResult.outputOptions)
 491
 492       addEncoderDefaultParams(localCommand, encoder)
 493
 494       streamsFound[stream] = encoder
 495       break
 496     }
 497
 498     if (!streamsFound[stream]) {
 499       throw new Error('No available encoder found ' + encodersToTry.join(', '))
 500     }
 501   }
 502
 503   localCommand.videoCodec(streamsFound.VIDEO)
 504
 505   return localCommand
 506 }
 507
 508 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 509   return command
 510     .format('mp4')
 511     .videoCodec('copy')
 512     .audioCodec('copy')
 513 }
 514
 515 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 516   return command
 517     .format('mp4')
 518     .audioCodec('copy')
 519     .noVideo()
 520 }
 521
 522 // ---------------------------------------------------------------------------
 523 // Utils
 524 // ---------------------------------------------------------------------------
 525
 526 function getFFmpeg (input: string) {
 527   // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems
 528   const command = ffmpeg(input, { niceness: FFMPEG_NICE.TRANSCODING, cwd: CONFIG.STORAGE.TMP_DIR })
 529
 530   if (CONFIG.TRANSCODING.THREADS > 0) {
 531     // If we don't set any threads ffmpeg will chose automatically
 532     command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
 533   }
 534
 535   return command
 536 }
 537
 538 async function runCommand (command: ffmpeg.FfmpegCommand, onEnd?: Function) {
 539   return new Promise<void>((res, rej) => {
 540     command.on('error', (err, stdout, stderr) => {
 541       if (onEnd) onEnd()
 542
 543       logger.error('Error in transcoding job.', { stdout, stderr })
 544       rej(err)
 545     })
 546
 547     command.on('end', () => {
 548       if (onEnd) onEnd()
 549
 550       res()
 551     })
 552
 553     command.run()
 554   })
 555 }