Add storyboard support

12 files changed, 124 insertions, 2 deletions

diff --git a/shared/ffmpeg/ffmpeg-images.ts b/shared/ffmpeg/ffmpeg-images.ts
index 2db63bd8b..27305382c 100644
--- a/shared/ffmpeg/ffmpeg-images.ts
+++ b/shared/ffmpeg/ffmpeg-images.ts
@@ -56,4 +56,41 @@ export class FFmpegImage {
         .thumbnail(thumbnailOptions)
     })
   }
+
+  async generateStoryboardFromVideo (options: {
+    path: string
+    destination: string
+
+    sprites: {
+      size: {
+        width: number
+        height: number
+      }
+
+      count: {
+        width: number
+        height: number
+      }
+
+      duration: number
+    }
+  }) {
+    const { path, destination, sprites } = options
+
+    const command = this.commandWrapper.buildCommand(path)
+
+    const filter = [
+      `setpts=N/round(FRAME_RATE)/TB`,
+      `select='not(mod(t,${options.sprites.duration}))'`,
+      `scale=${sprites.size.width}:${sprites.size.height}`,
+      `tile=layout=${sprites.count.width}x${sprites.count.height}`
+    ].join(',')
+
+    command.outputOption('-filter_complex', filter)
+    command.outputOption('-frames:v', '1')
+    command.outputOption('-q:v', '2')
+    command.output(destination)
+
+    return this.commandWrapper.runCommand()
+  }
 }
diff --git a/shared/models/activitypub/objects/index.ts b/shared/models/activitypub/objects/index.ts
index 9aa3c462c..a2e040b32 100644
--- a/shared/models/activitypub/objects/index.ts
+++ b/shared/models/activitypub/objects/index.ts
@@ -6,5 +6,5 @@ export * from './object.model'
 export * from './playlist-element-object'
 export * from './playlist-object'
 export * from './video-comment-object'
-export * from './video-torrent-object'
+export * from './video-object'
 export * from './watch-action-object'
diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-object.ts
index 23d54bdbd..a252a2df0 100644
--- a/shared/models/activitypub/objects/video-torrent-object.ts
+++ b/shared/models/activitypub/objects/video-object.ts
@@ -51,6 +51,22 @@ export interface VideoObject {
 
   attributedTo: ActivityPubAttributedTo[]
 
+  preview?: ActivityPubStoryboard[]
+
   to?: string[]
   cc?: string[]
 }
+
+export interface ActivityPubStoryboard {
+  type: 'Image'
+  rel: [ 'storyboard' ]
+  url: {
+    href: string
+    mediaType: string
+    width: number
+    height: number
+    tileWidth: number
+    tileHeight: number
+    tileDuration: string
+  }[]
+}
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts
index 4202589f3..1012312f3 100644
--- a/shared/models/server/custom-config.model.ts
+++ b/shared/models/server/custom-config.model.ts
@@ -78,6 +78,10 @@ export interface CustomConfig {
     torrents: {
       size: number
     }
+
+    storyboards: {
+      size: number
+    }
   }
 
   signup: {
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 22ecee324..9c40079fb 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -30,6 +30,7 @@ export type JobType =
   | 'video-studio-edition'
   | 'video-transcoding'
   | 'videos-views-stats'
+  | 'generate-video-storyboard'
 
 export interface Job {
   id: number | string
@@ -294,3 +295,10 @@ export interface TranscodingJobBuilderPayload {
     priority?: number
   }[][]
 }
+
+// ---------------------------------------------------------------------------
+
+export interface GenerateStoryboardPayload {
+  videoUUID: string
+  federate: boolean
+}
diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts
index 80be1854b..b3ce6ad3f 100644
--- a/shared/models/videos/index.ts
+++ b/shared/models/videos/index.ts
@@ -15,6 +15,7 @@ export * from './channel-sync'
 
 export * from './nsfw-policy.type'
 
+export * from './storyboard.model'
 export * from './thumbnail.type'
 
 export * from './video-constant.model'
diff --git a/shared/models/videos/storyboard.model.ts b/shared/models/videos/storyboard.model.ts
new file mode 100644
index 000000000..c92c81f09
--- /dev/null
+++ b/shared/models/videos/storyboard.model.ts
@@ -0,0 +1,11 @@
+export interface Storyboard {
+  storyboardPath: string
+
+  totalHeight: number
+  totalWidth: number
+
+  spriteHeight: number
+  spriteWidth: number
+
+  spriteDuration: number
+}
diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts
index b94bd2625..114db8091 100644
--- a/shared/server-commands/server/config-command.ts
+++ b/shared/server-commands/server/config-command.ts
@@ -159,6 +159,10 @@ export class ConfigCommand extends AbstractCommand {
       newConfig: {
         transcoding: {
           enabled: true,
+
+          allowAudioFiles: true,
+          allowAdditionalExtensions: true,
+
           resolutions: {
             ...ConfigCommand.getCustomConfigResolutions(false),
 
@@ -368,6 +372,9 @@ export class ConfigCommand extends AbstractCommand {
         },
         torrents: {
           size: 4
+        },
+        storyboards: {
+          size: 5
         }
       },
       signup: {
diff --git a/shared/server-commands/server/jobs.ts b/shared/server-commands/server/jobs.ts
index ff3098063..8f131fba4 100644
--- a/shared/server-commands/server/jobs.ts
+++ b/shared/server-commands/server/jobs.ts
@@ -33,6 +33,8 @@ async function waitJobs (
 
     // Check if each server has pending request
     for (const server of servers) {
+      if (process.env.DEBUG) console.log('Checking ' + server.url)
+
       for (const state of states) {
 
         const jobPromise = server.jobs.list({
@@ -45,6 +47,10 @@ async function waitJobs (
           .then(jobs => {
             if (jobs.length !== 0) {
               pendingRequests = true
+
+              if (process.env.DEBUG) {
+                console.log(jobs)
+              }
             }
           })
 
@@ -55,6 +61,10 @@ async function waitJobs (
         .then(obj => {
           if (obj.activityPubMessagesWaiting !== 0) {
             pendingRequests = true
+
+            if (process.env.DEBUG) {
+              console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting)
+            }
           }
         })
       tasks.push(debugPromise)
@@ -65,12 +75,15 @@ async function waitJobs (
             for (const job of data) {
               if (job.state.id !== RunnerJobState.COMPLETED) {
                 pendingRequests = true
+
+                if (process.env.DEBUG) {
+                  console.log(job)
+                }
               }
             }
           })
         tasks.push(runnerJobsPromise)
       }
-
     }
 
     return tasks
diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index 0911e22b0..6aa4296b0 100644
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -35,6 +35,7 @@ import {
   VideoPasswordsCommand,
   PlaylistsCommand,
   ServicesCommand,
+  StoryboardCommand,
   StreamingPlaylistsCommand,
   VideosCommand,
   VideoStudioCommand,
@@ -149,6 +150,8 @@ export class PeerTubeServer {
   registrations?: RegistrationsCommand
   videoPasswords?: VideoPasswordsCommand
 
+  storyboard?: StoryboardCommand
+
   runners?: RunnersCommand
   runnerRegistrationTokens?: RunnerRegistrationTokensCommand
   runnerJobs?: RunnerJobsCommand
@@ -436,6 +439,8 @@ export class PeerTubeServer {
     this.videoToken = new VideoTokenCommand(this)
     this.registrations = new RegistrationsCommand(this)
 
+    this.storyboard = new StoryboardCommand(this)
+
     this.runners = new RunnersCommand(this)
     this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this)
     this.runnerJobs = new RunnerJobsCommand(this)
diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts
index da36b5b6b..106d80af0 100644
--- a/shared/server-commands/videos/index.ts
+++ b/shared/server-commands/videos/index.ts
@@ -11,6 +11,7 @@ export * from './live-command'
 export * from './live'
 export * from './playlists-command'
 export * from './services-command'
+export * from './storyboard-command'
 export * from './streaming-playlists-command'
 export * from './comments-command'
 export * from './video-studio-command'
diff --git a/shared/server-commands/videos/storyboard-command.ts b/shared/server-commands/videos/storyboard-command.ts
new file mode 100644
index 000000000..06d90fc12
--- /dev/null
+++ b/shared/server-commands/videos/storyboard-command.ts
@@ -0,0 +1,19 @@
+import { HttpStatusCode, Storyboard } from '@shared/models'
+import { AbstractCommand, OverrideCommandOptions } from '../shared'
+
+export class StoryboardCommand extends AbstractCommand {
+
+  list (options: OverrideCommandOptions & {
+    id: number | string
+  }) {
+    const path = '/api/v1/videos/' + options.id + '/storyboards'
+
+    return this.getRequestBody<{ storyboards: Storyboard[] }>({
+      ...options,
+
+      path,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    })
+  }
+}