server/helpers/ffmpeg-utils.ts

   1 import * as ffmpeg from 'fluent-ffmpeg'
   2 import { dirname, join } from 'path'
   3 import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos'
   4 import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
   5 import { processImage } from './image-utils'
   6 import { logger } from './logger'
   7 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
   8 import { readFile, remove, writeFile } from 'fs-extra'
   9 import { CONFIG } from '../initializers/config'
  10 import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
  11
  12 /**
  13  * A toolbox to play with audio
  14  */
  15 namespace audio {
  16   export const get = (videoPath: string) => {
  17     // without position, ffprobe considers the last input only
  18     // we make it consider the first input only
  19     // if you pass a file path to pos, then ffprobe acts on that file directly
  20     return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
  21
  22       function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
  23         if (err) return rej(err)
  24
  25         if ('streams' in data) {
  26           const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
  27           if (audioStream) {
  28             return res({
  29               absolutePath: data.format.filename,
  30               audioStream
  31             })
  32           }
  33         }
  34
  35         return res({ absolutePath: data.format.filename })
  36       }
  37
  38       return ffmpeg.ffprobe(videoPath, parseFfprobe)
  39     })
  40   }
  41
  42   export namespace bitrate {
  43     const baseKbitrate = 384
  44
  45     const toBits = (kbits: number) => kbits * 8000
  46
  47     export const aac = (bitrate: number): number => {
  48       switch (true) {
  49         case bitrate > toBits(baseKbitrate):
  50           return baseKbitrate
  51
  52         default:
  53           return -1 // we interpret it as a signal to copy the audio stream as is
  54       }
  55     }
  56
  57     export const mp3 = (bitrate: number): number => {
  58       /*
  59       a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
  60       That's why, when using aac, we can go to lower kbit/sec. The equivalences
  61       made here are not made to be accurate, especially with good mp3 encoders.
  62       */
  63       switch (true) {
  64         case bitrate <= toBits(192):
  65           return 128
  66
  67         case bitrate <= toBits(384):
  68           return 256
  69
  70         default:
  71           return baseKbitrate
  72       }
  73     }
  74   }
  75 }
  76
  77 function computeResolutionsToTranscode (videoFileResolution: number) {
  78   const resolutionsEnabled: number[] = []
  79   const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
  80
  81   // Put in the order we want to proceed jobs
  82   const resolutions = [
  83     VideoResolution.H_NOVIDEO,
  84     VideoResolution.H_480P,
  85     VideoResolution.H_360P,
  86     VideoResolution.H_720P,
  87     VideoResolution.H_240P,
  88     VideoResolution.H_1080P,
  89     VideoResolution.H_4K
  90   ]
  91
  92   for (const resolution of resolutions) {
  93     if (configResolutions[resolution + 'p'] === true && videoFileResolution > resolution) {
  94       resolutionsEnabled.push(resolution)
  95     }
  96   }
  97
  98   return resolutionsEnabled
  99 }
 100
 101 async function getVideoStreamSize (path: string) {
 102   const videoStream = await getVideoStreamFromFile(path)
 103
 104   return videoStream === null
 105     ? { width: 0, height: 0 }
 106     : { width: videoStream.width, height: videoStream.height }
 107 }
 108
 109 async function getVideoStreamCodec (path: string) {
 110   const videoStream = await getVideoStreamFromFile(path)
 111
 112   if (!videoStream) return ''
 113
 114   const videoCodec = videoStream.codec_tag_string
 115
 116   const baseProfileMatrix = {
 117     High: '6400',
 118     Main: '4D40',
 119     Baseline: '42E0'
 120   }
 121
 122   let baseProfile = baseProfileMatrix[videoStream.profile]
 123   if (!baseProfile) {
 124     logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
 125     baseProfile = baseProfileMatrix['High'] // Fallback
 126   }
 127
 128   let level = videoStream.level.toString(16)
 129   if (level.length === 1) level = `0${level}`
 130
 131   return `${videoCodec}.${baseProfile}${level}`
 132 }
 133
 134 async function getAudioStreamCodec (path: string) {
 135   const { audioStream } = await audio.get(path)
 136
 137   if (!audioStream) return ''
 138
 139   const audioCodec = audioStream.codec_name
 140   if (audioCodec === 'aac') return 'mp4a.40.2'
 141
 142   logger.warn('Cannot get audio codec of %s.', path, { audioStream })
 143
 144   return 'mp4a.40.2' // Fallback
 145 }
 146
 147 async function getVideoFileResolution (path: string) {
 148   const size = await getVideoStreamSize(path)
 149
 150   return {
 151     videoFileResolution: Math.min(size.height, size.width),
 152     isPortraitMode: size.height > size.width
 153   }
 154 }
 155
 156 async function getVideoFileFPS (path: string) {
 157   const videoStream = await getVideoStreamFromFile(path)
 158   if (videoStream === null) return 0
 159
 160   for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
 161     const valuesText: string = videoStream[key]
 162     if (!valuesText) continue
 163
 164     const [ frames, seconds ] = valuesText.split('/')
 165     if (!frames || !seconds) continue
 166
 167     const result = parseInt(frames, 10) / parseInt(seconds, 10)
 168     if (result > 0) return Math.round(result)
 169   }
 170
 171   return 0
 172 }
 173
 174 async function getMetadataFromFile <T> (path: string, cb = metadata => metadata) {
 175   return new Promise<T>((res, rej) => {
 176     ffmpeg.ffprobe(path, (err, metadata) => {
 177       if (err) return rej(err)
 178
 179       return res(cb(new VideoFileMetadata(metadata)))
 180     })
 181   })
 182 }
 183
 184 async function getVideoFileBitrate (path: string) {
 185   return getMetadataFromFile<number>(path, metadata => metadata.format.bit_rate)
 186 }
 187
 188 function getDurationFromVideoFile (path: string) {
 189   return getMetadataFromFile<number>(path, metadata => Math.floor(metadata.format.duration))
 190 }
 191
 192 function getVideoStreamFromFile (path: string) {
 193   return getMetadataFromFile<any>(path, metadata => metadata.streams.find(s => s.codec_type === 'video') || null)
 194 }
 195
 196 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
 197   const pendingImageName = 'pending-' + imageName
 198
 199   const options = {
 200     filename: pendingImageName,
 201     count: 1,
 202     folder
 203   }
 204
 205   const pendingImagePath = join(folder, pendingImageName)
 206
 207   try {
 208     await new Promise<string>((res, rej) => {
 209       ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
 210         .on('error', rej)
 211         .on('end', () => res(imageName))
 212         .thumbnail(options)
 213     })
 214
 215     const destination = join(folder, imageName)
 216     await processImage(pendingImagePath, destination, size)
 217   } catch (err) {
 218     logger.error('Cannot generate image from video %s.', fromPath, { err })
 219
 220     try {
 221       await remove(pendingImagePath)
 222     } catch (err) {
 223       logger.debug('Cannot remove pending image path after generation error.', { err })
 224     }
 225   }
 226 }
 227
 228 type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
 229
 230 interface BaseTranscodeOptions {
 231   type: TranscodeOptionsType
 232   inputPath: string
 233   outputPath: string
 234   resolution: VideoResolution
 235   isPortraitMode?: boolean
 236 }
 237
 238 interface HLSTranscodeOptions extends BaseTranscodeOptions {
 239   type: 'hls'
 240   copyCodecs: boolean
 241   hlsPlaylist: {
 242     videoFilename: string
 243   }
 244 }
 245
 246 interface QuickTranscodeOptions extends BaseTranscodeOptions {
 247   type: 'quick-transcode'
 248 }
 249
 250 interface VideoTranscodeOptions extends BaseTranscodeOptions {
 251   type: 'video'
 252 }
 253
 254 interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
 255   type: 'merge-audio'
 256   audioPath: string
 257 }
 258
 259 interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
 260   type: 'only-audio'
 261 }
 262
 263 type TranscodeOptions =
 264   HLSTranscodeOptions
 265   | VideoTranscodeOptions
 266   | MergeAudioTranscodeOptions
 267   | OnlyAudioTranscodeOptions
 268   | QuickTranscodeOptions
 269
 270 function transcode (options: TranscodeOptions) {
 271   return new Promise<void>(async (res, rej) => {
 272     try {
 273       // we set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems
 274       let command = ffmpeg(options.inputPath, { niceness: FFMPEG_NICE.TRANSCODING, cwd: CONFIG.STORAGE.TMP_DIR })
 275         .output(options.outputPath)
 276
 277       if (options.type === 'quick-transcode') {
 278         command = buildQuickTranscodeCommand(command)
 279       } else if (options.type === 'hls') {
 280         command = await buildHLSCommand(command, options)
 281       } else if (options.type === 'merge-audio') {
 282         command = await buildAudioMergeCommand(command, options)
 283       } else if (options.type === 'only-audio') {
 284         command = buildOnlyAudioCommand(command, options)
 285       } else {
 286         command = await buildx264Command(command, options)
 287       }
 288
 289       if (CONFIG.TRANSCODING.THREADS > 0) {
 290         // if we don't set any threads ffmpeg will chose automatically
 291         command = command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
 292       }
 293
 294       command
 295         .on('error', (err, stdout, stderr) => {
 296           logger.error('Error in transcoding job.', { stdout, stderr })
 297           return rej(err)
 298         })
 299         .on('end', () => {
 300           return fixHLSPlaylistIfNeeded(options)
 301             .then(() => res())
 302             .catch(err => rej(err))
 303         })
 304         .run()
 305     } catch (err) {
 306       return rej(err)
 307     }
 308   })
 309 }
 310
 311 async function canDoQuickTranscode (path: string): Promise<boolean> {
 312   // NOTE: This could be optimized by running ffprobe only once (but it runs fast anyway)
 313   const videoStream = await getVideoStreamFromFile(path)
 314   const parsedAudio = await audio.get(path)
 315   const fps = await getVideoFileFPS(path)
 316   const bitRate = await getVideoFileBitrate(path)
 317   const resolution = await getVideoFileResolution(path)
 318
 319   // check video params
 320   if (videoStream == null) return false
 321   if (videoStream['codec_name'] !== 'h264') return false
 322   if (videoStream['pix_fmt'] !== 'yuv420p') return false
 323   if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
 324   if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false
 325
 326   // check audio params (if audio stream exists)
 327   if (parsedAudio.audioStream) {
 328     if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
 329
 330     const maxAudioBitrate = audio.bitrate['aac'](parsedAudio.audioStream['bit_rate'])
 331     if (maxAudioBitrate !== -1 && parsedAudio.audioStream['bit_rate'] > maxAudioBitrate) return false
 332   }
 333
 334   return true
 335 }
 336
 337 function getClosestFramerateStandard (fps: number, type: 'HD_STANDARD' | 'STANDARD'): number {
 338   return VIDEO_TRANSCODING_FPS[type].slice(0)
 339                                     .sort((a, b) => fps % a - fps % b)[0]
 340 }
 341
 342 function convertWebPToJPG (path: string, destination: string): Promise<void> {
 343   return new Promise<void>(async (res, rej) => {
 344     try {
 345       const command = ffmpeg(path).output(destination)
 346
 347       command.on('error', (err, stdout, stderr) => {
 348         logger.error('Error in ffmpeg webp convert process.', { stdout, stderr })
 349         return rej(err)
 350       })
 351       .on('end', () => res())
 352       .run()
 353     } catch (err) {
 354       return rej(err)
 355     }
 356   })
 357 }
 358
 359 // ---------------------------------------------------------------------------
 360
 361 export {
 362   getVideoStreamCodec,
 363   getAudioStreamCodec,
 364   convertWebPToJPG,
 365   getVideoStreamSize,
 366   getVideoFileResolution,
 367   getMetadataFromFile,
 368   getDurationFromVideoFile,
 369   generateImageFromVideoFile,
 370   TranscodeOptions,
 371   TranscodeOptionsType,
 372   transcode,
 373   getVideoFileFPS,
 374   computeResolutionsToTranscode,
 375   audio,
 376   getVideoFileBitrate,
 377   canDoQuickTranscode
 378 }
 379
 380 // ---------------------------------------------------------------------------
 381
 382 async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
 383   let fps = await getVideoFileFPS(options.inputPath)
 384   if (
 385     // On small/medium resolutions, limit FPS
 386     options.resolution !== undefined &&
 387     options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
 388     fps > VIDEO_TRANSCODING_FPS.AVERAGE
 389   ) {
 390     // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
 391     fps = getClosestFramerateStandard(fps, 'STANDARD')
 392   }
 393
 394   command = await presetH264(command, options.inputPath, options.resolution, fps)
 395
 396   if (options.resolution !== undefined) {
 397     // '?x720' or '720x?' for example
 398     const size = options.isPortraitMode === true ? `${options.resolution}x?` : `?x${options.resolution}`
 399     command = command.size(size)
 400   }
 401
 402   if (fps) {
 403     // Hard FPS limits
 404     if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
 405     else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
 406
 407     command = command.withFPS(fps)
 408   }
 409
 410   return command
 411 }
 412
 413 async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
 414   command = command.loop(undefined)
 415
 416   command = await presetH264VeryFast(command, options.audioPath, options.resolution)
 417
 418   command = command.input(options.audioPath)
 419                    .videoFilter('scale=trunc(iw/2)*2:trunc(ih/2)*2') // Avoid "height not divisible by 2" error
 420                    .outputOption('-tune stillimage')
 421                    .outputOption('-shortest')
 422
 423   return command
 424 }
 425
 426 function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) {
 427   command = presetOnlyAudio(command)
 428
 429   return command
 430 }
 431
 432 function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
 433   command = presetCopy(command)
 434
 435   command = command.outputOption('-map_metadata -1') // strip all metadata
 436                    .outputOption('-movflags faststart')
 437
 438   return command
 439 }
 440
 441 async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
 442   const videoPath = getHLSVideoPath(options)
 443
 444   if (options.copyCodecs) command = presetCopy(command)
 445   else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
 446   else command = await buildx264Command(command, options)
 447
 448   command = command.outputOption('-hls_time 4')
 449                    .outputOption('-hls_list_size 0')
 450                    .outputOption('-hls_playlist_type vod')
 451                    .outputOption('-hls_segment_filename ' + videoPath)
 452                    .outputOption('-hls_segment_type fmp4')
 453                    .outputOption('-f hls')
 454                    .outputOption('-hls_flags single_file')
 455
 456   return command
 457 }
 458
 459 function getHLSVideoPath (options: HLSTranscodeOptions) {
 460   return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
 461 }
 462
 463 async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
 464   if (options.type !== 'hls') return
 465
 466   const fileContent = await readFile(options.outputPath)
 467
 468   const videoFileName = options.hlsPlaylist.videoFilename
 469   const videoFilePath = getHLSVideoPath(options)
 470
 471   // Fix wrong mapping with some ffmpeg versions
 472   const newContent = fileContent.toString()
 473                                 .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
 474
 475   await writeFile(options.outputPath, newContent)
 476 }
 477
 478 /**
 479  * A slightly customised version of the 'veryfast' x264 preset
 480  *
 481  * The veryfast preset is right in the sweet spot of performance
 482  * and quality. Superfast and ultrafast will give you better
 483  * performance, but then quality is noticeably worse.
 484  */
 485 async function presetH264VeryFast (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 486   let localCommand = await presetH264(command, input, resolution, fps)
 487
 488   localCommand = localCommand.outputOption('-preset:v veryfast')
 489
 490   /*
 491   MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html
 492   Our target situation is closer to a livestream than a stream,
 493   since we want to reduce as much a possible the encoding burden,
 494   although not to the point of a livestream where there is a hard
 495   constraint on the frames per second to be encoded.
 496   */
 497
 498   return localCommand
 499 }
 500
 501 /**
 502  * Standard profile, with variable bitrate audio and faststart.
 503  *
 504  * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel
 505  * See https://trac.ffmpeg.org/wiki/Encode/AAC#fdk_vbr
 506  */
 507 async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolution: VideoResolution, fps?: number) {
 508   let localCommand = command
 509     .format('mp4')
 510     .videoCodec('libx264')
 511     .outputOption('-level 3.1') // 3.1 is the minimal resource allocation for our highest supported resolution
 512     .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
 513     .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
 514     .outputOption('-pix_fmt yuv420p') // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
 515     .outputOption('-map_metadata -1') // strip all metadata
 516     .outputOption('-movflags faststart')
 517
 518   const parsedAudio = await audio.get(input)
 519
 520   if (!parsedAudio.audioStream) {
 521     localCommand = localCommand.noAudio()
 522   } else if ((await checkFFmpegEncoders()).get('libfdk_aac')) { // we favor VBR, if a good AAC encoder is available
 523     localCommand = localCommand
 524       .audioCodec('libfdk_aac')
 525       .audioQuality(5)
 526   } else {
 527     // we try to reduce the ceiling bitrate by making rough matches of bitrates
 528     // of course this is far from perfect, but it might save some space in the end
 529     localCommand = localCommand.audioCodec('aac')
 530
 531     const audioCodecName = parsedAudio.audioStream['codec_name']
 532
 533     if (audio.bitrate[audioCodecName]) {
 534       const bitrate = audio.bitrate[audioCodecName](parsedAudio.audioStream['bit_rate'])
 535       if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate)
 536     }
 537   }
 538
 539   if (fps) {
 540     // Constrained Encoding (VBV)
 541     // https://slhck.info/video/2017/03/01/rate-control.html
 542     // https://trac.ffmpeg.org/wiki/Limiting%20the%20output%20bitrate
 543     const targetBitrate = getTargetBitrate(resolution, fps, VIDEO_TRANSCODING_FPS)
 544     localCommand = localCommand.outputOptions([ `-maxrate ${targetBitrate}`, `-bufsize ${targetBitrate * 2}` ])
 545
 546     // Keyframe interval of 2 seconds for faster seeking and resolution switching.
 547     // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
 548     // https://superuser.com/a/908325
 549     localCommand = localCommand.outputOption(`-g ${fps * 2}`)
 550   }
 551
 552   return localCommand
 553 }
 554
 555 function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 556   return command
 557     .format('mp4')
 558     .videoCodec('copy')
 559     .audioCodec('copy')
 560 }
 561
 562 function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
 563   return command
 564     .format('mp4')
 565     .audioCodec('copy')
 566     .noVideo()
 567 }