Add storyboard support

4 files changed, 255 insertions, 27 deletions

diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index f2001e432..4179545b8 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -5,6 +5,7 @@ import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
 import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 import { uuidToShort } from '@shared/extra-utils'
 import {
+  ActivityPubStoryboard,
   ActivityTagObject,
   ActivityUrlObject,
   Video,
@@ -347,29 +348,17 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
     name: t.name
   }))
 
-  let language
-  if (video.language) {
-    language = {
-      identifier: video.language,
-      name: getLanguageLabel(video.language)
-    }
-  }
+  const language = video.language
+    ? { identifier: video.language, name: getLanguageLabel(video.language) }
+    : undefined
 
-  let category
-  if (video.category) {
-    category = {
-      identifier: video.category + '',
-      name: getCategoryLabel(video.category)
-    }
-  }
+  const category = video.category
+    ? { identifier: video.category + '', name: getCategoryLabel(video.category) }
+    : undefined
 
-  let licence
-  if (video.licence) {
-    licence = {
-      identifier: video.licence + '',
-      name: getLicenceLabel(video.licence)
-    }
-  }
+  const licence = video.licence
+    ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) }
+    : undefined
 
   const url: ActivityUrlObject[] = [
     // HTML url should be the first element in the array so Mastodon correctly displays the embed
@@ -465,6 +454,8 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
       height: i.height
     })),
 
+    preview: buildPreviewAPAttribute(video),
+
     url,
 
     likes: getLocalVideoLikesActivityPubUrl(video),
@@ -541,3 +532,30 @@ function buildLiveAPAttributes (video: MVideoAP) {
     latencyMode: video.VideoLive.latencyMode
   }
 }
+
+function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {
+  if (!video.Storyboard) return undefined
+
+  const storyboard = video.Storyboard
+
+  return [
+    {
+      type: 'Image',
+      rel: [ 'storyboard' ],
+      url: [
+        {
+          mediaType: 'image/jpeg',
+
+          href: storyboard.getOriginFileUrl(video),
+
+          width: storyboard.totalWidth,
+          height: storyboard.totalHeight,
+
+          tileWidth: storyboard.spriteWidth,
+          tileHeight: storyboard.spriteHeight,
+          tileDuration: getActivityStreamDuration(storyboard.spriteDuration)
+        }
+      ]
+    }
+  ]
+}
diff --git a/server/models/video/storyboard.ts b/server/models/video/storyboard.ts
new file mode 100644
index 000000000..65a044c98
--- /dev/null
+++ b/server/models/video/storyboard.ts
@@ -0,0 +1,169 @@
+import { remove } from 'fs-extra'
+import { join } from 'path'
+import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { CONFIG } from '@server/initializers/config'
+import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models'
+import { Storyboard } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { logger } from '../../helpers/logger'
+import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
+import { VideoModel } from './video'
+import { Transaction } from 'sequelize'
+
+@Table({
+  tableName: 'storyboard',
+  indexes: [
+    {
+      fields: [ 'videoId' ],
+      unique: true
+    },
+    {
+      fields: [ 'filename' ],
+      unique: true
+    }
+  ]
+})
+export class StoryboardModel extends Model<Partial<AttributesOnly<StoryboardModel>>> {
+
+  @AllowNull(false)
+  @Column
+  filename: string
+
+  @AllowNull(false)
+  @Column
+  totalHeight: number
+
+  @AllowNull(false)
+  @Column
+  totalWidth: number
+
+  @AllowNull(false)
+  @Column
+  spriteHeight: number
+
+  @AllowNull(false)
+  @Column
+  spriteWidth: number
+
+  @AllowNull(false)
+  @Column
+  spriteDuration: number
+
+  @AllowNull(true)
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max))
+  fileUrl: string
+
+  @ForeignKey(() => VideoModel)
+  @Column
+  videoId: number
+
+  @BelongsTo(() => VideoModel, {
+    foreignKey: {
+      allowNull: true
+    },
+    onDelete: 'CASCADE'
+  })
+  Video: VideoModel
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @AfterDestroy
+  static removeInstanceFile (instance: StoryboardModel) {
+    logger.info('Removing storyboard file %s.', instance.filename)
+
+    // Don't block the transaction
+    instance.removeFile()
+            .catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err }))
+  }
+
+  static loadByVideo (videoId: number, transaction?: Transaction): Promise<MStoryboard> {
+    const query = {
+      where: {
+        videoId
+      },
+      transaction
+    }
+
+    return StoryboardModel.findOne(query)
+  }
+
+  static loadByFilename (filename: string): Promise<MStoryboard> {
+    const query = {
+      where: {
+        filename
+      }
+    }
+
+    return StoryboardModel.findOne(query)
+  }
+
+  static loadWithVideoByFilename (filename: string): Promise<MStoryboardVideo> {
+    const query = {
+      where: {
+        filename
+      },
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: true
+        }
+      ]
+    }
+
+    return StoryboardModel.findOne(query)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  static async listStoryboardsOf (video: MVideo): Promise<MStoryboardVideo[]> {
+    const query = {
+      where: {
+        videoId: video.id
+      }
+    }
+
+    const storyboards = await StoryboardModel.findAll<MStoryboard>(query)
+
+    return storyboards.map(s => Object.assign(s, { Video: video }))
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getOriginFileUrl (video: MVideo) {
+    if (video.isOwned()) {
+      return WEBSERVER.URL + this.getLocalStaticPath()
+    }
+
+    return this.fileUrl
+  }
+
+  getLocalStaticPath () {
+    return LAZY_STATIC_PATHS.STORYBOARDS + this.filename
+  }
+
+  getPath () {
+    return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename)
+  }
+
+  removeFile () {
+    return remove(this.getPath())
+  }
+
+  toFormattedJSON (this: MStoryboardVideo): Storyboard {
+    return {
+      storyboardPath: this.getLocalStaticPath(),
+
+      totalHeight: this.totalHeight,
+      totalWidth: this.totalWidth,
+
+      spriteWidth: this.spriteWidth,
+      spriteHeight: this.spriteHeight,
+
+      spriteDuration: this.spriteDuration
+    }
+  }
+}
diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts
index 1fb1cae82..dd4cefd65 100644
--- a/server/models/video/video-caption.ts
+++ b/server/models/video/video-caption.ts
@@ -15,7 +15,7 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models'
+import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, MVideoCaptionVideo } from '@server/types/models'
 import { buildUUID } from '@shared/extra-utils'
 import { AttributesOnly } from '@shared/typescript-utils'
 import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
@@ -225,7 +225,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
     }
   }
 
-  getCaptionStaticPath (this: MVideoCaption) {
+  getCaptionStaticPath (this: MVideoCaptionLanguageUrl) {
     return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename)
   }
 
@@ -233,9 +233,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
     return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename)
   }
 
-  getFileUrl (video: MVideo) {
-    if (!this.Video) this.Video = video as VideoModel
-
+  getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideo) {
     if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath()
 
     return this.fileUrl
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index f90f2b7f6..0e9a84426 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -58,7 +58,7 @@ import {
 import { AttributesOnly } from '@shared/typescript-utils'
 import { peertubeTruncate } from '../../helpers/core-utils'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
+import { exists, isArray, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
 import {
   isVideoDescriptionValid,
   isVideoDurationValid,
@@ -75,6 +75,7 @@ import {
   MChannel,
   MChannelAccountDefault,
   MChannelId,
+  MStoryboard,
   MStreamingPlaylist,
   MStreamingPlaylistFilesVideo,
   MUserAccountId,
@@ -83,6 +84,8 @@ import {
   MVideoAccountLight,
   MVideoAccountLightBlacklistAllFiles,
   MVideoAP,
+  MVideoAPLight,
+  MVideoCaptionLanguageUrl,
   MVideoDetails,
   MVideoFileVideo,
   MVideoFormattable,
@@ -126,6 +129,7 @@ import {
   VideosIdListQueryBuilder,
   VideosModelListQueryBuilder
 } from './sql/video'
+import { StoryboardModel } from './storyboard'
 import { TagModel } from './tag'
 import { ThumbnailModel } from './thumbnail'
 import { VideoBlacklistModel } from './video-blacklist'
@@ -753,6 +757,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
   })
   VideoJobInfo: VideoJobInfoModel
 
+  @HasOne(() => StoryboardModel, {
+    foreignKey: {
+      name: 'videoId',
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  Storyboard: StoryboardModel
+
   @AfterCreate
   static notifyCreate (video: MVideo) {
     InternalEventEmitter.Instance.emit('video-created', { video })
@@ -904,6 +917,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
           required: false
         },
         {
+          model: StoryboardModel.unscoped(),
+          required: false
+        },
+        {
           attributes: [ 'id', 'url' ],
           model: VideoShareModel.unscoped(),
           required: false,
@@ -1768,6 +1785,32 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
     )
   }
 
+  async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise<MVideoAP> {
+    const videoAP = this as MVideoAP
+
+    const getCaptions = () => {
+      if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions
+
+      return this.$get('VideoCaptions', {
+        attributes: [ 'filename', 'language', 'fileUrl' ],
+        transaction
+      }) as Promise<MVideoCaptionLanguageUrl[]>
+    }
+
+    const getStoryboard = () => {
+      if (videoAP.Storyboard) return videoAP.Storyboard
+
+      return this.$get('Storyboard', { transaction }) as Promise<MStoryboard>
+    }
+
+    const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ])
+
+    return Object.assign(this, {
+      VideoCaptions: captions,
+      Storyboard: storyboard
+    })
+  }
+
   getTruncatedDescription () {
     if (!this.description) return null