Add storyboard support

5 files changed, 46 insertions, 11 deletions

diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts
index bd0c54b0c..d7e251153 100644
--- a/server/lib/activitypub/videos/federate.ts
+++ b/server/lib/activitypub/videos/federate.ts
@@ -1,10 +1,9 @@
 import { Transaction } from 'sequelize/types'
-import { isArray } from '@server/helpers/custom-validators/misc'
-import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models'
+import { MVideoAP, MVideoAPLight } from '@server/types/models'
 import { sendCreateVideo, sendUpdateVideo } from '../send'
 import { shareVideoByServerAndChannel } from '../share'
 
-async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
+async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) {
   const video = videoArg as MVideoAP
 
   if (
@@ -13,13 +12,7 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid
     // Check the video is public/unlisted and published
     video.hasPrivacyForFederation() && video.hasStateForFederation()
   ) {
-    // Fetch more attributes that we will need to serialize in AP object
-    if (isArray(video.VideoCaptions) === false) {
-      video.VideoCaptions = await video.$get('VideoCaptions', {
-        attributes: [ 'filename', 'language' ],
-        transaction
-      })
-    }
+    const video = await videoArg.lightAPToFullAP(transaction)
 
     if (isNewVideo) {
       // Now we'll add the video's meta data to our followers
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
index c0b92c93d..7c5c73139 100644
--- a/server/lib/activitypub/videos/shared/abstract-builder.ts
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -3,6 +3,7 @@ import { deleteAllModels, filterNonExistingModels } from '@server/helpers/databa
 import { logger, LoggerTagsFn } from '@server/helpers/logger'
 import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
 import { setVideoTags } from '@server/lib/video'
+import { StoryboardModel } from '@server/models/video/storyboard'
 import { VideoCaptionModel } from '@server/models/video/video-caption'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { VideoLiveModel } from '@server/models/video/video-live'
@@ -24,6 +25,7 @@ import {
   getFileAttributesFromUrl,
   getLiveAttributesFromObject,
   getPreviewFromIcons,
+  getStoryboardAttributeFromObject,
   getStreamingPlaylistAttributesFromObject,
   getTagsFromObject,
   getThumbnailFromIcons
@@ -107,6 +109,16 @@ export abstract class APVideoAbstractBuilder {
     }
   }
 
+  protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) {
+    const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t)
+    if (existingStoryboard) await existingStoryboard.destroy({ transaction: t })
+
+    const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject)
+    if (!storyboardAttributes) return
+
+    return StoryboardModel.create(storyboardAttributes, { transaction: t })
+  }
+
   protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) {
     const attributes = getLiveAttributesFromObject(video, this.videoObject)
     const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true })
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts
index 77321d8a5..e6d7bc23c 100644
--- a/server/lib/activitypub/videos/shared/creator.ts
+++ b/server/lib/activitypub/videos/shared/creator.ts
@@ -48,6 +48,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
         await this.setTrackers(videoCreated, t)
         await this.insertOrReplaceCaptions(videoCreated, t)
         await this.insertOrReplaceLive(videoCreated, t)
+        await this.insertOrReplaceStoryboard(videoCreated, t)
 
         // We added a video in this channel, set it as updated
         await channel.setAsUpdated(t)
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index 8fd0a816c..a9e0bed97 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -1,6 +1,6 @@
 import { maxBy, minBy } from 'lodash'
 import { decode as magnetUriDecode } from 'magnet-uri'
-import { basename } from 'path'
+import { basename, extname } from 'path'
 import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos'
 import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos'
 import { logger } from '@server/helpers/logger'
@@ -25,6 +25,9 @@ import {
   VideoStreamingPlaylistType
 } from '@shared/models'
 import { getDurationFromActivityStream } from '../../activity'
+import { isArray } from '@server/helpers/custom-validators/misc'
+import { generateImageFilename } from '@server/helpers/image-utils'
+import { arrayify } from '@shared/core-utils'
 
 function getThumbnailFromIcons (videoObject: VideoObject) {
   let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@@ -166,6 +169,26 @@ function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObje
   }))
 }
 
+function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) {
+  if (!isArray(videoObject.preview)) return undefined
+
+  const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard'))
+  if (!storyboard) return undefined
+
+  const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg')
+
+  return {
+    filename: generateImageFilename(extname(url.href)),
+    totalHeight: url.height,
+    totalWidth: url.width,
+    spriteHeight: url.tileHeight,
+    spriteWidth: url.tileWidth,
+    spriteDuration: getDurationFromActivityStream(url.tileDuration),
+    fileUrl: url.href,
+    videoId: video.id
+  }
+}
+
 function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
     ? VideoPrivacy.PUBLIC
@@ -228,6 +251,7 @@ export {
 
   getLiveAttributesFromObject,
   getCaptionAttributesFromObject,
+  getStoryboardAttributeFromObject,
 
   getVideoAttributesFromObject
 }
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts
index 6ddd2301b..3a0886523 100644
--- a/server/lib/activitypub/videos/updater.ts
+++ b/server/lib/activitypub/videos/updater.ts
@@ -57,6 +57,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
       await Promise.all([
         runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
         runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
+        runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)),
         this.setOrDeleteLive(videoUpdated),
         this.setPreview(videoUpdated)
       ])
@@ -138,6 +139,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
     await this.insertOrReplaceCaptions(videoUpdated, t)
   }
 
+  private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) {
+    await this.insertOrReplaceStoryboard(videoUpdated, t)
+  }
+
   private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
     if (!this.video.isLive) return