From d8f39b126d9fe4bec1c12fb213548cc6edc87867 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 1 Jun 2023 14:51:16 +0200 Subject: Add storyboard support --- .../lib/job-queue/handlers/generate-storyboard.ts | 138 +++++++++++++++++++++ server/lib/job-queue/handlers/video-import.ts | 9 ++ server/lib/job-queue/handlers/video-live-ending.ts | 20 ++- server/lib/job-queue/job-queue.ts | 11 +- 4 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 server/lib/job-queue/handlers/generate-storyboard.ts (limited to 'server/lib/job-queue') diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts new file mode 100644 index 000000000..652cac272 --- /dev/null +++ b/server/lib/job-queue/handlers/generate-storyboard.ts @@ -0,0 +1,138 @@ +import { Job } from 'bullmq' +import { join } from 'path' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' +import { generateImageFilename, getImageSize } from '@server/helpers/image-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { STORYBOARD } from '@server/initializers/constants' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' +import { VideoPathManager } from '@server/lib/video-path-manager' +import { StoryboardModel } from '@server/models/video/storyboard' +import { VideoModel } from '@server/models/video/video' +import { MVideo } from '@server/types/models' +import { FFmpegImage, isAudioFile } from '@shared/ffmpeg' +import { GenerateStoryboardPayload } from '@shared/models' + +const lTagsBase = loggerTagsFactory('storyboard') + +async function processGenerateStoryboard (job: Job): Promise { + const payload = job.data as GenerateStoryboardPayload + const lTags = lTagsBase(payload.videoUUID) + + logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags) + + const video = await VideoModel.loadFull(payload.videoUUID) + if (!video) { + logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) + return + } + + const inputFile = video.getMaxQualityFile() + + await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => { + const isAudio = await isAudioFile(videoPath) + + if (isAudio) { + logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags) + return + } + + const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')) + + const filename = generateImageFilename() + const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) + + const totalSprites = buildTotalSprites(video) + const spriteDuration = Math.round(video.duration / totalSprites) + + const spritesCount = findGridSize({ + toFind: totalSprites, + maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT + }) + + logger.debug( + 'Generating storyboard from video of %s to %s', video.uuid, destination, + { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration } + ) + + await ffmpeg.generateStoryboardFromVideo({ + destination, + path: videoPath, + sprites: { + size: STORYBOARD.SPRITE_SIZE, + count: spritesCount, + duration: spriteDuration + } + }) + + const imageSize = await getImageSize(destination) + + const existing = await StoryboardModel.loadByVideo(video.id) + if (existing) await existing.destroy() + + await StoryboardModel.create({ + filename, + totalHeight: imageSize.height, + totalWidth: imageSize.width, + spriteHeight: STORYBOARD.SPRITE_SIZE.height, + spriteWidth: STORYBOARD.SPRITE_SIZE.width, + spriteDuration, + videoId: video.id + }) + + logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags) + }) + + if (payload.federate) { + await federateVideoIfNeeded(video, false) + } +} + +// --------------------------------------------------------------------------- + +export { + processGenerateStoryboard +} + +function buildTotalSprites (video: MVideo) { + const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width + const totalSprites = Math.min(Math.ceil(video.duration), maxSprites) + + // We can generate a single line + if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites + + return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) +} + +function findGridSize (options: { + toFind: number + maxEdgeCount: number +}) { + const { toFind, maxEdgeCount } = options + + for (let i = 1; i <= maxEdgeCount; i++) { + for (let j = i; j <= maxEdgeCount; j++) { + if (toFind === i * j) return { width: j, height: i } + } + } + + throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`) +} + +function findGridFit (value: number, maxMultiplier: number) { + for (let i = value; i--; i > 0) { + if (!isPrimeWithin(i, maxMultiplier)) return i + } + + throw new Error('Could not find prime number below ' + value) +} + +function isPrimeWithin (value: number, maxMultiplier: number) { + if (value < 2) return false + + for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) { + if (value % i === 0 && value / i <= maxMultiplier) return false + } + + return true +} diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index cdd362f6e..c1355dcef 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -306,6 +306,15 @@ async function afterImportSuccess (options: { Notifier.Instance.notifyOnNewVideoIfNeeded(video) } + // Generate the storyboard in the job queue, and don't forget to federate an update after + await JobQueue.Instance.createJob({ + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + federate: true + } + }) + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { await JobQueue.Instance.createJob( await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 49feb53f2..95d4f5e64 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -1,6 +1,8 @@ import { Job } from 'bullmq' import { readdir, remove } from 'fs-extra' import { join } from 'path' +import { peertubeTruncate } from '@server/helpers/core-utils' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' @@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { peertubeTruncate } from '@server/helpers/core-utils' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' +import { JobQueue } from '../job-queue' const lTags = loggerTagsFactory('live', 'job') @@ -147,6 +148,8 @@ async function saveReplayToExternalVideo (options: { } await moveToNextState({ video: replayVideo, isNewVideo: true }) + + await createStoryboardJob(replayVideo) } async function replaceLiveByReplay (options: { @@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: { await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) + // FIXME: should not happen in this function if (permanentLive) { // Remove session replay await remove(replayDirectory) } else { // We won't stream again in this live, we can delete the base replay directory @@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: { // We consider this is a new video await moveToNextState({ video: videoWithFiles, isNewVideo: true }) + + await createStoryboardJob(videoWithFiles) } async function assignReplayFilesToVideo (options: { @@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: { logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) } } + +function createStoryboardJob (video: MVideo) { + return JobQueue.Instance.createJob({ + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + federate: true + } + }) +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 03f6fbea7..177bca285 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -25,6 +25,7 @@ import { DeleteResumableUploadMetaFilePayload, EmailPayload, FederateVideoPayload, + GenerateStoryboardPayload, JobState, JobType, ManageVideoTorrentPayload, @@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending' import { processVideoStudioEdition } from './handlers/video-studio-edition' import { processVideoTranscoding } from './handlers/video-transcoding' import { processVideosViewsStats } from './handlers/video-views-stats' +import { processGenerateStoryboard } from './handlers/generate-storyboard' export type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -91,7 +93,8 @@ export type CreateJobArgument = { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | { type: 'notify', payload: NotifyPayload } | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | - { type: 'federate-video', payload: FederateVideoPayload } + { type: 'federate-video', payload: FederateVideoPayload } | + { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } export type CreateJobOptions = { delay?: number @@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise } = { 'video-redundancy': processVideoRedundancy, 'video-studio-edition': processVideoStudioEdition, 'video-transcoding': processVideoTranscoding, - 'videos-views-stats': processVideosViewsStats + 'videos-views-stats': processVideosViewsStats, + 'generate-video-storyboard': processGenerateStoryboard } const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } = { @@ -141,10 +145,11 @@ const jobTypes: JobType[] = [ 'after-video-channel-import', 'email', 'federate-video', - 'transcoding-job-builder', + 'generate-video-storyboard', 'manage-video-torrent', 'move-to-object-storage', 'notify', + 'transcoding-job-builder', 'video-channel-import', 'video-file-import', 'video-import', -- cgit v1.2.3