4 files changed, 173 insertions, 5 deletions

diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts
new file mode 100644
index 000000000..652cac272
--- /dev/null
+++ b/server/lib/job-queue/handlers/generate-storyboard.ts
@@ -0,0 +1,138 @@
+import { Job } from 'bullmq'
+import { join } from 'path'
+import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
+import { generateImageFilename, getImageSize } from '@server/helpers/image-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { STORYBOARD } from '@server/initializers/constants'
+import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { VideoModel } from '@server/models/video/video'
+import { MVideo } from '@server/types/models'
+import { FFmpegImage, isAudioFile } from '@shared/ffmpeg'
+import { GenerateStoryboardPayload } from '@shared/models'
+
+const lTagsBase = loggerTagsFactory('storyboard')
+
+async function processGenerateStoryboard (job: Job): Promise<void> {
+  const payload = job.data as GenerateStoryboardPayload
+  const lTags = lTagsBase(payload.videoUUID)
+
+  logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags)
+
+  const video = await VideoModel.loadFull(payload.videoUUID)
+  if (!video) {
+    logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags)
+    return
+  }
+
+  const inputFile = video.getMaxQualityFile()
+
+  await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
+    const isAudio = await isAudioFile(videoPath)
+
+    if (isAudio) {
+      logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
+      return
+    }
+
+    const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
+
+    const filename = generateImageFilename()
+    const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
+
+    const totalSprites = buildTotalSprites(video)
+    const spriteDuration = Math.round(video.duration / totalSprites)
+
+    const spritesCount = findGridSize({
+      toFind: totalSprites,
+      maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
+    })
+
+    logger.debug(
+      'Generating storyboard from video of %s to %s', video.uuid, destination,
+      { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration }
+    )
+
+    await ffmpeg.generateStoryboardFromVideo({
+      destination,
+      path: videoPath,
+      sprites: {
+        size: STORYBOARD.SPRITE_SIZE,
+        count: spritesCount,
+        duration: spriteDuration
+      }
+    })
+
+    const imageSize = await getImageSize(destination)
+
+    const existing = await StoryboardModel.loadByVideo(video.id)
+    if (existing) await existing.destroy()
+
+    await StoryboardModel.create({
+      filename,
+      totalHeight: imageSize.height,
+      totalWidth: imageSize.width,
+      spriteHeight: STORYBOARD.SPRITE_SIZE.height,
+      spriteWidth: STORYBOARD.SPRITE_SIZE.width,
+      spriteDuration,
+      videoId: video.id
+    })
+
+    logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags)
+  })
+
+  if (payload.federate) {
+    await federateVideoIfNeeded(video, false)
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processGenerateStoryboard
+}
+
+function buildTotalSprites (video: MVideo) {
+  const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width
+  const totalSprites = Math.min(Math.ceil(video.duration), maxSprites)
+
+  // We can generate a single line
+  if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites
+
+  return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT)
+}
+
+function findGridSize (options: {
+  toFind: number
+  maxEdgeCount: number
+}) {
+  const { toFind, maxEdgeCount } = options
+
+  for (let i = 1; i <= maxEdgeCount; i++) {
+    for (let j = i; j <= maxEdgeCount; j++) {
+      if (toFind === i * j) return { width: j, height: i }
+    }
+  }
+
+  throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
+}
+
+function findGridFit (value: number, maxMultiplier: number) {
+  for (let i = value; i--; i > 0) {
+    if (!isPrimeWithin(i, maxMultiplier)) return i
+  }
+
+  throw new Error('Could not find prime number below ' + value)
+}
+
+function isPrimeWithin (value: number, maxMultiplier: number) {
+  if (value < 2) return false
+
+  for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) {
+    if (value % i === 0 && value / i <= maxMultiplier) return false
+  }
+
+  return true
+}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index cdd362f6e..c1355dcef 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -306,6 +306,15 @@ async function afterImportSuccess (options: {
     Notifier.Instance.notifyOnNewVideoIfNeeded(video)
   }
 
+  // Generate the storyboard in the job queue, and don't forget to federate an update after
+  await JobQueue.Instance.createJob({
+    type: 'generate-video-storyboard' as 'generate-video-storyboard',
+    payload: {
+      videoUUID: video.uuid,
+      federate: true
+    }
+  })
+
   if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
     await JobQueue.Instance.createJob(
       await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 49feb53f2..95d4f5e64 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -1,6 +1,8 @@
 import { Job } from 'bullmq'
 import { readdir, remove } from 'fs-extra'
 import { join } from 'path'
+import { peertubeTruncate } from '@server/helpers/core-utils'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
 import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
@@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv
 import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'
 import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'
-import { peertubeTruncate } from '@server/helpers/core-utils'
-import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
+import { JobQueue } from '../job-queue'
 
 const lTags = loggerTagsFactory('live', 'job')
 
@@ -147,6 +148,8 @@ async function saveReplayToExternalVideo (options: {
   }
 
   await moveToNextState({ video: replayVideo, isNewVideo: true })
+
+  await createStoryboardJob(replayVideo)
 }
 
 async function replaceLiveByReplay (options: {
@@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: {
 
   await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
 
+  // FIXME: should not happen in this function
   if (permanentLive) { // Remove session replay
     await remove(replayDirectory)
   } else { // We won't stream again in this live, we can delete the base replay directory
@@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: {
 
   // We consider this is a new video
   await moveToNextState({ video: videoWithFiles, isNewVideo: true })
+
+  await createStoryboardJob(videoWithFiles)
 }
 
 async function assignReplayFilesToVideo (options: {
@@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: {
     logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
   }
 }
+
+function createStoryboardJob (video: MVideo) {
+  return JobQueue.Instance.createJob({
+    type: 'generate-video-storyboard' as 'generate-video-storyboard',
+    payload: {
+      videoUUID: video.uuid,
+      federate: true
+    }
+  })
+}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 03f6fbea7..177bca285 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -25,6 +25,7 @@ import {
   DeleteResumableUploadMetaFilePayload,
   EmailPayload,
   FederateVideoPayload,
+  GenerateStoryboardPayload,
   JobState,
   JobType,
   ManageVideoTorrentPayload,
@@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending'
 import { processVideoStudioEdition } from './handlers/video-studio-edition'
 import { processVideoTranscoding } from './handlers/video-transcoding'
 import { processVideosViewsStats } from './handlers/video-views-stats'
+import { processGenerateStoryboard } from './handlers/generate-storyboard'
 
 export type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -91,7 +93,8 @@ export type CreateJobArgument =
   { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
   { type: 'notify', payload: NotifyPayload } |
   { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
-  { type: 'federate-video', payload: FederateVideoPayload }
+  { type: 'federate-video', payload: FederateVideoPayload } |
+  { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
 
 export type CreateJobOptions = {
   delay?: number
@@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
   'video-redundancy': processVideoRedundancy,
   'video-studio-edition': processVideoStudioEdition,
   'video-transcoding': processVideoTranscoding,
-  'videos-views-stats': processVideosViewsStats
+  'videos-views-stats': processVideosViewsStats,
+  'generate-video-storyboard': processGenerateStoryboard
 }
 
 const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
@@ -141,10 +145,11 @@ const jobTypes: JobType[] = [
   'after-video-channel-import',
   'email',
   'federate-video',
-  'transcoding-job-builder',
+  'generate-video-storyboard',
   'manage-video-torrent',
   'move-to-object-storage',
   'notify',
+  'transcoding-job-builder',
   'video-channel-import',
   'video-file-import',
   'video-import',