From c729caf6cc34630877a0e5a1bda1719384cd0c8a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 11 Feb 2022 10:51:33 +0100 Subject: Add basic video editor support --- server/helpers/ffmpeg/ffmpeg-commons.ts | 114 ++++++++++++++ server/helpers/ffmpeg/ffmpeg-edition.ts | 242 +++++++++++++++++++++++++++++ server/helpers/ffmpeg/ffmpeg-encoders.ts | 116 ++++++++++++++ server/helpers/ffmpeg/ffmpeg-images.ts | 46 ++++++ server/helpers/ffmpeg/ffmpeg-live.ts | 161 ++++++++++++++++++++ server/helpers/ffmpeg/ffmpeg-presets.ts | 156 +++++++++++++++++++ server/helpers/ffmpeg/ffmpeg-vod.ts | 254 +++++++++++++++++++++++++++++++ server/helpers/ffmpeg/ffprobe-utils.ts | 231 ++++++++++++++++++++++++++++ server/helpers/ffmpeg/index.ts | 8 + 9 files changed, 1328 insertions(+) create mode 100644 server/helpers/ffmpeg/ffmpeg-commons.ts create mode 100644 server/helpers/ffmpeg/ffmpeg-edition.ts create mode 100644 server/helpers/ffmpeg/ffmpeg-encoders.ts create mode 100644 server/helpers/ffmpeg/ffmpeg-images.ts create mode 100644 server/helpers/ffmpeg/ffmpeg-live.ts create mode 100644 server/helpers/ffmpeg/ffmpeg-presets.ts create mode 100644 server/helpers/ffmpeg/ffmpeg-vod.ts create mode 100644 server/helpers/ffmpeg/ffprobe-utils.ts create mode 100644 server/helpers/ffmpeg/index.ts (limited to 'server/helpers/ffmpeg') diff --git a/server/helpers/ffmpeg/ffmpeg-commons.ts b/server/helpers/ffmpeg/ffmpeg-commons.ts new file mode 100644 index 000000000..ee338889c --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-commons.ts @@ -0,0 +1,114 @@ +import { Job } from 'bull' +import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg' +import { execPromise } from '@server/helpers/core-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { FFMPEG_NICE } from '@server/initializers/constants' +import { EncoderOptions } from '@shared/models' + +const lTags = loggerTagsFactory('ffmpeg') + +type StreamType = 'audio' | 'video' + +function getFFmpeg (input: string, type: 'live' | 'vod') { + // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems + const command = ffmpeg(input, { + niceness: type === 'live' ? FFMPEG_NICE.LIVE : FFMPEG_NICE.VOD, + cwd: CONFIG.STORAGE.TMP_DIR + }) + + const threads = type === 'live' + ? CONFIG.LIVE.TRANSCODING.THREADS + : CONFIG.TRANSCODING.THREADS + + if (threads > 0) { + // If we don't set any threads ffmpeg will chose automatically + command.outputOption('-threads ' + threads) + } + + return command +} + +function getFFmpegVersion () { + return new Promise((res, rej) => { + (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => { + if (err) return rej(err) + if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path')) + + return execPromise(`${ffmpegPath} -version`) + .then(stdout => { + const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/) + if (!parsed || !parsed[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`)) + + // Fix ffmpeg version that does not include patch version (4.4 for example) + let version = parsed[1] + if (version.match(/^\d+\.\d+$/)) { + version += '.0' + } + + return res(version) + }) + .catch(err => rej(err)) + }) + }) +} + +async function runCommand (options: { + command: FfmpegCommand + silent?: boolean // false by default + job?: Job +}) { + const { command, silent = false, job } = options + + return new Promise((res, rej) => { + let shellCommand: string + + command.on('start', cmdline => { shellCommand = cmdline }) + + command.on('error', (err, stdout, stderr) => { + if (silent !== true) logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...lTags() }) + + rej(err) + }) + + command.on('end', (stdout, stderr) => { + logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...lTags() }) + + res() + }) + + if (job) { + command.on('progress', progress => { + if (!progress.percent) return + + job.progress(Math.round(progress.percent)) + .catch(err => logger.warn('Cannot set ffmpeg job progress.', { err, ...lTags() })) + }) + } + + command.run() + }) +} + +function buildStreamSuffix (base: string, streamNum?: number) { + if (streamNum !== undefined) { + return `${base}:${streamNum}` + } + + return base +} + +function getScaleFilter (options: EncoderOptions): string { + if (options.scaleFilter) return options.scaleFilter.name + + return 'scale' +} + +export { + getFFmpeg, + getFFmpegVersion, + runCommand, + StreamType, + buildStreamSuffix, + getScaleFilter +} diff --git a/server/helpers/ffmpeg/ffmpeg-edition.ts b/server/helpers/ffmpeg/ffmpeg-edition.ts new file mode 100644 index 000000000..a5baa7ef1 --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-edition.ts @@ -0,0 +1,242 @@ +import { FilterSpecification } from 'fluent-ffmpeg' +import { VIDEO_FILTERS } from '@server/initializers/constants' +import { AvailableEncoders } from '@shared/models' +import { logger, loggerTagsFactory } from '../logger' +import { getFFmpeg, runCommand } from './ffmpeg-commons' +import { presetCopy, presetVOD } from './ffmpeg-presets' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe-utils' + +const lTags = loggerTagsFactory('ffmpeg') + +async function cutVideo (options: { + inputPath: string + outputPath: string + start?: number + end?: number +}) { + const { inputPath, outputPath } = options + + logger.debug('Will cut the video.', { options, ...lTags() }) + + let command = getFFmpeg(inputPath, 'vod') + .output(outputPath) + + command = presetCopy(command) + + if (options.start) command.inputOption('-ss ' + options.start) + + if (options.end) { + const endSeeking = options.end - (options.start || 0) + + command.outputOption('-to ' + endSeeking) + } + + await runCommand({ command }) +} + +async function addWatermark (options: { + inputPath: string + watermarkPath: string + outputPath: string + + availableEncoders: AvailableEncoders + profile: string +}) { + const { watermarkPath, inputPath, outputPath, availableEncoders, profile } = options + + logger.debug('Will add watermark to the video.', { options, ...lTags() }) + + const videoProbe = await ffprobePromise(inputPath) + const fps = await getVideoStreamFPS(inputPath, videoProbe) + const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe) + + let command = getFFmpeg(inputPath, 'vod') + .output(outputPath) + command.input(watermarkPath) + + command = await presetVOD({ + command, + input: inputPath, + availableEncoders, + profile, + resolution, + fps, + canCopyAudio: true, + canCopyVideo: false + }) + + const complexFilter: FilterSpecification[] = [ + // Scale watermark + { + inputs: [ '[1]', '[0]' ], + filter: 'scale2ref', + options: { + w: 'oh*mdar', + h: `ih*${VIDEO_FILTERS.WATERMARK.SIZE_RATIO}` + }, + outputs: [ '[watermark]', '[video]' ] + }, + + { + inputs: [ '[video]', '[watermark]' ], + filter: 'overlay', + options: { + x: `main_w - overlay_w - (main_h * ${VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO})`, + y: `main_h * ${VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO}` + } + } + ] + + command.complexFilter(complexFilter) + + await runCommand({ command }) +} + +async function addIntroOutro (options: { + inputPath: string + introOutroPath: string + outputPath: string + type: 'intro' | 'outro' + + availableEncoders: AvailableEncoders + profile: string +}) { + const { introOutroPath, inputPath, outputPath, availableEncoders, profile, type } = options + + logger.debug('Will add intro/outro to the video.', { options, ...lTags() }) + + const mainProbe = await ffprobePromise(inputPath) + const fps = await getVideoStreamFPS(inputPath, mainProbe) + const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) + const mainHasAudio = await hasAudioStream(inputPath, mainProbe) + + const introOutroProbe = await ffprobePromise(introOutroPath) + const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe) + + let command = getFFmpeg(inputPath, 'vod') + .output(outputPath) + + command.input(introOutroPath) + + if (!introOutroHasAudio && mainHasAudio) { + const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe) + + command.input('anullsrc') + command.withInputFormat('lavfi') + command.withInputOption('-t ' + duration) + } + + command = await presetVOD({ + command, + input: inputPath, + availableEncoders, + profile, + resolution, + fps, + canCopyAudio: false, + canCopyVideo: false + }) + + // Add black background to correctly scale intro/outro with padding + const complexFilter: FilterSpecification[] = [ + { + inputs: [ '1', '0' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: `ih` + }, + outputs: [ 'intro-outro', 'main' ] + }, + { + inputs: [ 'intro-outro', 'main' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: `ih` + }, + outputs: [ 'to-scale', 'main' ] + }, + { + inputs: 'to-scale', + filter: 'drawbox', + options: { + t: 'fill' + }, + outputs: [ 'to-scale-bg' ] + }, + { + inputs: [ '1', 'to-scale-bg' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: 'ih', + force_original_aspect_ratio: 'decrease', + flags: 'spline' + }, + outputs: [ 'to-scale', 'to-scale-bg' ] + }, + { + inputs: [ 'to-scale-bg', 'to-scale' ], + filter: 'overlay', + options: { + x: '(main_w - overlay_w)/2', + y: '(main_h - overlay_h)/2' + }, + outputs: 'intro-outro-resized' + } + ] + + const concatFilter = { + inputs: [], + filter: 'concat', + options: { + n: 2, + v: 1, + unsafe: 1 + }, + outputs: [ 'v' ] + } + + const introOutroFilterInputs = [ 'intro-outro-resized' ] + const mainFilterInputs = [ 'main' ] + + if (mainHasAudio) { + mainFilterInputs.push('0:a') + + if (introOutroHasAudio) { + introOutroFilterInputs.push('1:a') + } else { + // Silent input + introOutroFilterInputs.push('2:a') + } + } + + if (type === 'intro') { + concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ] + } else { + concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ] + } + + if (mainHasAudio) { + concatFilter.options['a'] = 1 + concatFilter.outputs.push('a') + + command.outputOption('-map [a]') + } + + command.outputOption('-map [v]') + + complexFilter.push(concatFilter) + command.complexFilter(complexFilter) + + await runCommand({ command }) +} + +// --------------------------------------------------------------------------- + +export { + cutVideo, + addIntroOutro, + addWatermark +} diff --git a/server/helpers/ffmpeg/ffmpeg-encoders.ts b/server/helpers/ffmpeg/ffmpeg-encoders.ts new file mode 100644 index 000000000..5bd80ba05 --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-encoders.ts @@ -0,0 +1,116 @@ +import { getAvailableEncoders } from 'fluent-ffmpeg' +import { pick } from '@shared/core-utils' +import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models' +import { promisify0 } from '../core-utils' +import { logger, loggerTagsFactory } from '../logger' + +const lTags = loggerTagsFactory('ffmpeg') + +// Detect supported encoders by ffmpeg +let supportedEncoders: Map +async function checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise> { + if (supportedEncoders !== undefined) { + return supportedEncoders + } + + const getAvailableEncodersPromise = promisify0(getAvailableEncoders) + const availableFFmpegEncoders = await getAvailableEncodersPromise() + + const searchEncoders = new Set() + for (const type of [ 'live', 'vod' ]) { + for (const streamType of [ 'audio', 'video' ]) { + for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) { + searchEncoders.add(encoder) + } + } + } + + supportedEncoders = new Map() + + for (const searchEncoder of searchEncoders) { + supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined) + } + + logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...lTags() }) + + return supportedEncoders +} + +function resetSupportedEncoders () { + supportedEncoders = undefined +} + +// Run encoder builder depending on available encoders +// Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one +// If the default one does not exist, check the next encoder +async function getEncoderBuilderResult (options: EncoderOptionsBuilderParams & { + streamType: 'video' | 'audio' + input: string + + availableEncoders: AvailableEncoders + profile: string + + videoType: 'vod' | 'live' +}) { + const { availableEncoders, profile, streamType, videoType } = options + + const encodersToTry = availableEncoders.encodersToTry[videoType][streamType] + const encoders = availableEncoders.available[videoType] + + for (const encoder of encodersToTry) { + if (!(await checkFFmpegEncoders(availableEncoders)).get(encoder)) { + logger.debug('Encoder %s not available in ffmpeg, skipping.', encoder, lTags()) + continue + } + + if (!encoders[encoder]) { + logger.debug('Encoder %s not available in peertube encoders, skipping.', encoder, lTags()) + continue + } + + // An object containing available profiles for this encoder + const builderProfiles: EncoderProfile = encoders[encoder] + let builder = builderProfiles[profile] + + if (!builder) { + logger.debug('Profile %s for encoder %s not available. Fallback to default.', profile, encoder, lTags()) + builder = builderProfiles.default + + if (!builder) { + logger.debug('Default profile for encoder %s not available. Try next available encoder.', encoder, lTags()) + continue + } + } + + const result = await builder( + pick(options, [ + 'input', + 'canCopyAudio', + 'canCopyVideo', + 'resolution', + 'inputBitrate', + 'fps', + 'inputRatio', + 'streamNum' + ]) + ) + + return { + result, + + // If we don't have output options, then copy the input stream + encoder: result.copy === true + ? 'copy' + : encoder + } + } + + return null +} + +export { + checkFFmpegEncoders, + resetSupportedEncoders, + + getEncoderBuilderResult +} diff --git a/server/helpers/ffmpeg/ffmpeg-images.ts b/server/helpers/ffmpeg/ffmpeg-images.ts new file mode 100644 index 000000000..7f64c6d0a --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-images.ts @@ -0,0 +1,46 @@ +import ffmpeg from 'fluent-ffmpeg' +import { FFMPEG_NICE } from '@server/initializers/constants' +import { runCommand } from './ffmpeg-commons' + +function convertWebPToJPG (path: string, destination: string): Promise { + const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL }) + .output(destination) + + return runCommand({ command, silent: true }) +} + +function processGIF ( + path: string, + destination: string, + newSize: { width: number, height: number } +): Promise { + const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL }) + .fps(20) + .size(`${newSize.width}x${newSize.height}`) + .output(destination) + + return runCommand({ command }) +} + +async function generateThumbnailFromVideo (fromPath: string, folder: string, imageName: string) { + const pendingImageName = 'pending-' + imageName + + const options = { + filename: pendingImageName, + count: 1, + folder + } + + return new Promise((res, rej) => { + ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL }) + .on('error', rej) + .on('end', () => res(imageName)) + .thumbnail(options) + }) +} + +export { + convertWebPToJPG, + processGIF, + generateThumbnailFromVideo +} diff --git a/server/helpers/ffmpeg/ffmpeg-live.ts b/server/helpers/ffmpeg/ffmpeg-live.ts new file mode 100644 index 000000000..ff571626c --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-live.ts @@ -0,0 +1,161 @@ +import { FfmpegCommand, FilterSpecification } from 'fluent-ffmpeg' +import { join } from 'path' +import { VIDEO_LIVE } from '@server/initializers/constants' +import { AvailableEncoders } from '@shared/models' +import { logger, loggerTagsFactory } from '../logger' +import { buildStreamSuffix, getFFmpeg, getScaleFilter, StreamType } from './ffmpeg-commons' +import { getEncoderBuilderResult } from './ffmpeg-encoders' +import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './ffmpeg-presets' +import { computeFPS } from './ffprobe-utils' + +const lTags = loggerTagsFactory('ffmpeg') + +async function getLiveTranscodingCommand (options: { + inputUrl: string + + outPath: string + masterPlaylistName: string + + resolutions: number[] + + // Input information + fps: number + bitrate: number + ratio: number + + availableEncoders: AvailableEncoders + profile: string +}) { + const { inputUrl, outPath, resolutions, fps, bitrate, availableEncoders, profile, masterPlaylistName, ratio } = options + + const command = getFFmpeg(inputUrl, 'live') + + const varStreamMap: string[] = [] + + const complexFilter: FilterSpecification[] = [ + { + inputs: '[v:0]', + filter: 'split', + options: resolutions.length, + outputs: resolutions.map(r => `vtemp${r}`) + } + ] + + command.outputOption('-sc_threshold 0') + + addDefaultEncoderGlobalParams(command) + + for (let i = 0; i < resolutions.length; i++) { + const resolution = resolutions[i] + const resolutionFPS = computeFPS(fps, resolution) + + const baseEncoderBuilderParams = { + input: inputUrl, + + availableEncoders, + profile, + + canCopyAudio: true, + canCopyVideo: true, + + inputBitrate: bitrate, + inputRatio: ratio, + + resolution, + fps: resolutionFPS, + + streamNum: i, + videoType: 'live' as 'live' + } + + { + const streamType: StreamType = 'video' + const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) + if (!builderResult) { + throw new Error('No available live video encoder found') + } + + command.outputOption(`-map [vout${resolution}]`) + + addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i }) + + logger.debug( + 'Apply ffmpeg live video params from %s using %s profile.', builderResult.encoder, profile, + { builderResult, fps: resolutionFPS, resolution, ...lTags() } + ) + + command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`) + applyEncoderOptions(command, builderResult.result) + + complexFilter.push({ + inputs: `vtemp${resolution}`, + filter: getScaleFilter(builderResult.result), + options: `w=-2:h=${resolution}`, + outputs: `vout${resolution}` + }) + } + + { + const streamType: StreamType = 'audio' + const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) + if (!builderResult) { + throw new Error('No available live audio encoder found') + } + + command.outputOption('-map a:0') + + addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i }) + + logger.debug( + 'Apply ffmpeg live audio params from %s using %s profile.', builderResult.encoder, profile, + { builderResult, fps: resolutionFPS, resolution, ...lTags() } + ) + + command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`) + applyEncoderOptions(command, builderResult.result) + } + + varStreamMap.push(`v:${i},a:${i}`) + } + + command.complexFilter(complexFilter) + + addDefaultLiveHLSParams(command, outPath, masterPlaylistName) + + command.outputOption('-var_stream_map', varStreamMap.join(' ')) + + return command +} + +function getLiveMuxingCommand (inputUrl: string, outPath: string, masterPlaylistName: string) { + const command = getFFmpeg(inputUrl, 'live') + + command.outputOption('-c:v copy') + command.outputOption('-c:a copy') + command.outputOption('-map 0:a?') + command.outputOption('-map 0:v?') + + addDefaultLiveHLSParams(command, outPath, masterPlaylistName) + + return command +} + +// --------------------------------------------------------------------------- + +export { + getLiveTranscodingCommand, + getLiveMuxingCommand +} + +// --------------------------------------------------------------------------- + +function addDefaultLiveHLSParams (command: FfmpegCommand, outPath: string, masterPlaylistName: string) { + command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS) + command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE) + command.outputOption('-hls_flags delete_segments+independent_segments') + command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) + command.outputOption('-master_pl_name ' + masterPlaylistName) + command.outputOption(`-f hls`) + + command.output(join(outPath, '%v.m3u8')) +} diff --git a/server/helpers/ffmpeg/ffmpeg-presets.ts b/server/helpers/ffmpeg/ffmpeg-presets.ts new file mode 100644 index 000000000..99b39f79a --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-presets.ts @@ -0,0 +1,156 @@ +import { FfmpegCommand } from 'fluent-ffmpeg' +import { pick } from 'lodash' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { AvailableEncoders, EncoderOptions } from '@shared/models' +import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-commons' +import { getEncoderBuilderResult } from './ffmpeg-encoders' +import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from './ffprobe-utils' + +const lTags = loggerTagsFactory('ffmpeg') + +// --------------------------------------------------------------------------- + +function addDefaultEncoderGlobalParams (command: FfmpegCommand) { + // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375 + command.outputOption('-max_muxing_queue_size 1024') + // strip all metadata + .outputOption('-map_metadata -1') + // allows import of source material with incompatible pixel formats (e.g. MJPEG video) + .outputOption('-pix_fmt yuv420p') +} + +function addDefaultEncoderParams (options: { + command: FfmpegCommand + encoder: 'libx264' | string + fps: number + + streamNum?: number +}) { + const { command, encoder, fps, streamNum } = options + + if (encoder === 'libx264') { + // 3.1 is the minimal resource allocation for our highest supported resolution + command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1') + + if (fps) { + // Keyframe interval of 2 seconds for faster seeking and resolution switching. + // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html + // https://superuser.com/a/908325 + command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2)) + } + } +} + +// --------------------------------------------------------------------------- + +async function presetVOD (options: { + command: FfmpegCommand + input: string + + availableEncoders: AvailableEncoders + profile: string + + canCopyAudio: boolean + canCopyVideo: boolean + + resolution: number + fps: number + + scaleFilterValue?: string +}) { + const { command, input, profile, resolution, fps, scaleFilterValue } = options + + let localCommand = command + .format('mp4') + .outputOption('-movflags faststart') + + addDefaultEncoderGlobalParams(command) + + const probe = await ffprobePromise(input) + + // Audio encoder + const bitrate = await getVideoStreamBitrate(input, probe) + const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe) + + let streamsToProcess: StreamType[] = [ 'audio', 'video' ] + + if (!await hasAudioStream(input, probe)) { + localCommand = localCommand.noAudio() + streamsToProcess = [ 'video' ] + } + + for (const streamType of streamsToProcess) { + const builderResult = await getEncoderBuilderResult({ + ...pick(options, [ 'availableEncoders', 'canCopyAudio', 'canCopyVideo' ]), + + input, + inputBitrate: bitrate, + inputRatio: videoStreamDimensions?.ratio || 0, + + profile, + resolution, + fps, + streamType, + + videoType: 'vod' as 'vod' + }) + + if (!builderResult) { + throw new Error('No available encoder found for stream ' + streamType) + } + + logger.debug( + 'Apply ffmpeg params from %s for %s stream of input %s using %s profile.', + builderResult.encoder, streamType, input, profile, + { builderResult, resolution, fps, ...lTags() } + ) + + if (streamType === 'video') { + localCommand.videoCodec(builderResult.encoder) + + if (scaleFilterValue) { + localCommand.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`) + } + } else if (streamType === 'audio') { + localCommand.audioCodec(builderResult.encoder) + } + + applyEncoderOptions(localCommand, builderResult.result) + addDefaultEncoderParams({ command: localCommand, encoder: builderResult.encoder, fps }) + } + + return localCommand +} + +function presetCopy (command: FfmpegCommand): FfmpegCommand { + return command + .format('mp4') + .videoCodec('copy') + .audioCodec('copy') +} + +function presetOnlyAudio (command: FfmpegCommand): FfmpegCommand { + return command + .format('mp4') + .audioCodec('copy') + .noVideo() +} + +function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions): FfmpegCommand { + return command + .inputOptions(options.inputOptions ?? []) + .outputOptions(options.outputOptions ?? []) +} + +// --------------------------------------------------------------------------- + +export { + presetVOD, + presetCopy, + presetOnlyAudio, + + addDefaultEncoderGlobalParams, + addDefaultEncoderParams, + + applyEncoderOptions +} diff --git a/server/helpers/ffmpeg/ffmpeg-vod.ts b/server/helpers/ffmpeg/ffmpeg-vod.ts new file mode 100644 index 000000000..c3622ceb1 --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-vod.ts @@ -0,0 +1,254 @@ +import { Job } from 'bull' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { readFile, writeFile } from 'fs-extra' +import { dirname } from 'path' +import { pick } from '@shared/core-utils' +import { AvailableEncoders, VideoResolution } from '@shared/models' +import { logger, loggerTagsFactory } from '../logger' +import { getFFmpeg, runCommand } from './ffmpeg-commons' +import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets' +import { computeFPS, getVideoStreamFPS } from './ffprobe-utils' +import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' + +const lTags = loggerTagsFactory('ffmpeg') + +// --------------------------------------------------------------------------- + +type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio' + +interface BaseTranscodeVODOptions { + type: TranscodeVODOptionsType + + inputPath: string + outputPath: string + + availableEncoders: AvailableEncoders + profile: string + + resolution: number + + isPortraitMode?: boolean + + job?: Job +} + +interface HLSTranscodeOptions extends BaseTranscodeVODOptions { + type: 'hls' + copyCodecs: boolean + hlsPlaylist: { + videoFilename: string + } +} + +interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions { + type: 'hls-from-ts' + + isAAC: boolean + + hlsPlaylist: { + videoFilename: string + } +} + +interface QuickTranscodeOptions extends BaseTranscodeVODOptions { + type: 'quick-transcode' +} + +interface VideoTranscodeOptions extends BaseTranscodeVODOptions { + type: 'video' +} + +interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions { + type: 'merge-audio' + audioPath: string +} + +interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions { + type: 'only-audio' +} + +type TranscodeVODOptions = + HLSTranscodeOptions + | HLSFromTSTranscodeOptions + | VideoTranscodeOptions + | MergeAudioTranscodeOptions + | OnlyAudioTranscodeOptions + | QuickTranscodeOptions + +// --------------------------------------------------------------------------- + +const builders: { + [ type in TranscodeVODOptionsType ]: (c: FfmpegCommand, o?: TranscodeVODOptions) => Promise | FfmpegCommand +} = { + 'quick-transcode': buildQuickTranscodeCommand, + 'hls': buildHLSVODCommand, + 'hls-from-ts': buildHLSVODFromTSCommand, + 'merge-audio': buildAudioMergeCommand, + 'only-audio': buildOnlyAudioCommand, + 'video': buildVODCommand +} + +async function transcodeVOD (options: TranscodeVODOptions) { + logger.debug('Will run transcode.', { options, ...lTags() }) + + let command = getFFmpeg(options.inputPath, 'vod') + .output(options.outputPath) + + command = await builders[options.type](command, options) + + await runCommand({ command, job: options.job }) + + await fixHLSPlaylistIfNeeded(options) +} + +// --------------------------------------------------------------------------- + +export { + transcodeVOD, + + buildVODCommand, + + TranscodeVODOptions, + TranscodeVODOptionsType +} + +// --------------------------------------------------------------------------- + +async function buildVODCommand (command: FfmpegCommand, options: TranscodeVODOptions) { + let fps = await getVideoStreamFPS(options.inputPath) + fps = computeFPS(fps, options.resolution) + + let scaleFilterValue: string + + if (options.resolution !== undefined) { + scaleFilterValue = options.isPortraitMode === true + ? `w=${options.resolution}:h=-2` + : `w=-2:h=${options.resolution}` + } + + command = await presetVOD({ + ...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]), + + command, + input: options.inputPath, + canCopyAudio: true, + canCopyVideo: true, + fps, + scaleFilterValue + }) + + return command +} + +function buildQuickTranscodeCommand (command: FfmpegCommand) { + command = presetCopy(command) + + command = command.outputOption('-map_metadata -1') // strip all metadata + .outputOption('-movflags faststart') + + return command +} + +// --------------------------------------------------------------------------- +// Audio transcoding +// --------------------------------------------------------------------------- + +async function buildAudioMergeCommand (command: FfmpegCommand, options: MergeAudioTranscodeOptions) { + command = command.loop(undefined) + + const scaleFilterValue = getMergeAudioScaleFilterValue() + command = await presetVOD({ + ...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]), + + command, + input: options.audioPath, + canCopyAudio: true, + canCopyVideo: true, + fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, + scaleFilterValue + }) + + command.outputOption('-preset:v veryfast') + + command = command.input(options.audioPath) + .outputOption('-tune stillimage') + .outputOption('-shortest') + + return command +} + +function buildOnlyAudioCommand (command: FfmpegCommand, _options: OnlyAudioTranscodeOptions) { + command = presetOnlyAudio(command) + + return command +} + +// --------------------------------------------------------------------------- +// HLS transcoding +// --------------------------------------------------------------------------- + +async function buildHLSVODCommand (command: FfmpegCommand, options: HLSTranscodeOptions) { + const videoPath = getHLSVideoPath(options) + + if (options.copyCodecs) command = presetCopy(command) + else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command) + else command = await buildVODCommand(command, options) + + addCommonHLSVODCommandOptions(command, videoPath) + + return command +} + +function buildHLSVODFromTSCommand (command: FfmpegCommand, options: HLSFromTSTranscodeOptions) { + const videoPath = getHLSVideoPath(options) + + command.outputOption('-c copy') + + if (options.isAAC) { + // Required for example when copying an AAC stream from an MPEG-TS + // Since it's a bitstream filter, we don't need to reencode the audio + command.outputOption('-bsf:a aac_adtstoasc') + } + + addCommonHLSVODCommandOptions(command, videoPath) + + return command +} + +function addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) { + return command.outputOption('-hls_time 4') + .outputOption('-hls_list_size 0') + .outputOption('-hls_playlist_type vod') + .outputOption('-hls_segment_filename ' + outputPath) + .outputOption('-hls_segment_type fmp4') + .outputOption('-f hls') + .outputOption('-hls_flags single_file') +} + +async function fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) { + if (options.type !== 'hls' && options.type !== 'hls-from-ts') return + + const fileContent = await readFile(options.outputPath) + + const videoFileName = options.hlsPlaylist.videoFilename + const videoFilePath = getHLSVideoPath(options) + + // Fix wrong mapping with some ffmpeg versions + const newContent = fileContent.toString() + .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`) + + await writeFile(options.outputPath, newContent) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) { + return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` +} + +// Avoid "height not divisible by 2" error +function getMergeAudioScaleFilterValue () { + return 'trunc(iw/2)*2:trunc(ih/2)*2' +} diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts new file mode 100644 index 000000000..07bcf01f4 --- /dev/null +++ b/server/helpers/ffmpeg/ffprobe-utils.ts @@ -0,0 +1,231 @@ +import { FfprobeData } from 'fluent-ffmpeg' +import { getMaxBitrate } from '@shared/core-utils' +import { + ffprobePromise, + getAudioStream, + getVideoStreamDuration, + getMaxAudioBitrate, + buildFileMetadata, + getVideoStreamBitrate, + getVideoStreamFPS, + getVideoStream, + getVideoStreamDimensionsInfo, + hasAudioStream +} from '@shared/extra-utils/ffprobe' +import { VideoResolution, VideoTranscodingFPS } from '@shared/models' +import { CONFIG } from '../../initializers/config' +import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants' +import { logger } from '../logger' + +/** + * + * Helpers to run ffprobe and extract data from the JSON output + * + */ + +// --------------------------------------------------------------------------- +// Codecs +// --------------------------------------------------------------------------- + +async function getVideoStreamCodec (path: string) { + const videoStream = await getVideoStream(path) + if (!videoStream) return '' + + const videoCodec = videoStream.codec_tag_string + + if (videoCodec === 'vp09') return 'vp09.00.50.08' + if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0' + + const baseProfileMatrix = { + avc1: { + High: '6400', + Main: '4D40', + Baseline: '42E0' + }, + av01: { + High: '1', + Main: '0', + Professional: '2' + } + } + + let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile] + if (!baseProfile) { + logger.warn('Cannot get video profile codec of %s.', path, { videoStream }) + baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback + } + + if (videoCodec === 'av01') { + const level = videoStream.level + + // Guess the tier indicator and bit depth + return `${videoCodec}.${baseProfile}.${level}M.08` + } + + // Default, h264 codec + let level = videoStream.level.toString(16) + if (level.length === 1) level = `0${level}` + + return `${videoCodec}.${baseProfile}${level}` +} + +async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { + const { audioStream } = await getAudioStream(path, existingProbe) + + if (!audioStream) return '' + + const audioCodecName = audioStream.codec_name + + if (audioCodecName === 'opus') return 'opus' + if (audioCodecName === 'vorbis') return 'vorbis' + if (audioCodecName === 'aac') return 'mp4a.40.2' + + logger.warn('Cannot get audio codec of %s.', path, { audioStream }) + + return 'mp4a.40.2' // Fallback +} + +// --------------------------------------------------------------------------- +// Resolutions +// --------------------------------------------------------------------------- + +function computeLowerResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') { + const configResolutions = type === 'vod' + ? CONFIG.TRANSCODING.RESOLUTIONS + : CONFIG.LIVE.TRANSCODING.RESOLUTIONS + + const resolutionsEnabled: number[] = [] + + // Put in the order we want to proceed jobs + const resolutions: VideoResolution[] = [ + VideoResolution.H_NOVIDEO, + VideoResolution.H_480P, + VideoResolution.H_360P, + VideoResolution.H_720P, + VideoResolution.H_240P, + VideoResolution.H_144P, + VideoResolution.H_1080P, + VideoResolution.H_1440P, + VideoResolution.H_4K + ] + + for (const resolution of resolutions) { + if (configResolutions[resolution + 'p'] === true && videoFileResolution > resolution) { + resolutionsEnabled.push(resolution) + } + } + + return resolutionsEnabled +} + +// --------------------------------------------------------------------------- +// Can quick transcode +// --------------------------------------------------------------------------- + +async function canDoQuickTranscode (path: string): Promise { + if (CONFIG.TRANSCODING.PROFILE !== 'default') return false + + const probe = await ffprobePromise(path) + + return await canDoQuickVideoTranscode(path, probe) && + await canDoQuickAudioTranscode(path, probe) +} + +async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { + const parsedAudio = await getAudioStream(path, probe) + + if (!parsedAudio.audioStream) return true + + if (parsedAudio.audioStream['codec_name'] !== 'aac') return false + + const audioBitrate = parsedAudio.bitrate + if (!audioBitrate) return false + + const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) + if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false + + const channelLayout = parsedAudio.audioStream['channel_layout'] + // Causes playback issues with Chrome + if (!channelLayout || channelLayout === 'unknown') return false + + return true +} + +async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise { + const videoStream = await getVideoStream(path, probe) + const fps = await getVideoStreamFPS(path, probe) + const bitRate = await getVideoStreamBitrate(path, probe) + const resolutionData = await getVideoStreamDimensionsInfo(path, probe) + + // If ffprobe did not manage to guess the bitrate + if (!bitRate) return false + + // check video params + if (!videoStream) return false + if (videoStream['codec_name'] !== 'h264') return false + if (videoStream['pix_fmt'] !== 'yuv420p') return false + if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false + if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false + + return true +} + +// --------------------------------------------------------------------------- +// Framerate +// --------------------------------------------------------------------------- + +function getClosestFramerateStandard > (fps: number, type: K) { + return VIDEO_TRANSCODING_FPS[type].slice(0) + .sort((a, b) => fps % a - fps % b)[0] +} + +function computeFPS (fpsArg: number, resolution: VideoResolution) { + let fps = fpsArg + + if ( + // On small/medium resolutions, limit FPS + resolution !== undefined && + resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && + fps > VIDEO_TRANSCODING_FPS.AVERAGE + ) { + // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value + fps = getClosestFramerateStandard(fps, 'STANDARD') + } + + // Hard FPS limits + if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD') + + if (fps < VIDEO_TRANSCODING_FPS.MIN) { + throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`) + } + + return fps +} + +// --------------------------------------------------------------------------- + +export { + // Re export ffprobe utils + getVideoStreamDimensionsInfo, + buildFileMetadata, + getMaxAudioBitrate, + getVideoStream, + getVideoStreamDuration, + getAudioStream, + hasAudioStream, + getVideoStreamFPS, + ffprobePromise, + getVideoStreamBitrate, + + getVideoStreamCodec, + getAudioStreamCodec, + + computeFPS, + getClosestFramerateStandard, + + computeLowerResolutionsToTranscode, + + canDoQuickTranscode, + canDoQuickVideoTranscode, + canDoQuickAudioTranscode +} diff --git a/server/helpers/ffmpeg/index.ts b/server/helpers/ffmpeg/index.ts new file mode 100644 index 000000000..e3bb2013f --- /dev/null +++ b/server/helpers/ffmpeg/index.ts @@ -0,0 +1,8 @@ +export * from './ffmpeg-commons' +export * from './ffmpeg-edition' +export * from './ffmpeg-encoders' +export * from './ffmpeg-images' +export * from './ffmpeg-live' +export * from './ffmpeg-presets' +export * from './ffmpeg-vod' +export * from './ffprobe-utils' -- cgit v1.2.3