5 files changed, 216 insertions, 1 deletions

diff --git a/client/src/assets/player/peertube-player-manager.ts b/client/src/assets/player/peertube-player-manager.ts
index 2781850b9..66d9c7298 100644
--- a/client/src/assets/player/peertube-player-manager.ts
+++ b/client/src/assets/player/peertube-player-manager.ts
@@ -6,6 +6,7 @@ import './shared/stats/stats-plugin'
 import './shared/bezels/bezels-plugin'
 import './shared/peertube/peertube-plugin'
 import './shared/resolutions/peertube-resolutions-plugin'
+import './shared/control-bar/storyboard-plugin'
 import './shared/control-bar/next-previous-video-button'
 import './shared/control-bar/p2p-info-button'
 import './shared/control-bar/peertube-link-button'
@@ -42,6 +43,12 @@ CaptionsButton.prototype.controlText_ = 'Subtitles/CC'
 // We just want to display 'Off' instead of 'captions off', keep a space so the variable == true (hacky I know)
 CaptionsButton.prototype.label_ = ' '
 
+// TODO: remove when https://github.com/videojs/video.js/pull/7598 is merged
+const PlayProgressBar = videojs.getComponent('PlayProgressBar') as any
+if (PlayProgressBar.prototype.options_.children.includes('timeTooltip') !== true) {
+  PlayProgressBar.prototype.options_.children.push('timeTooltip')
+}
+
 export class PeertubePlayerManager {
   private static playerElementClassName: string
   private static playerElementAttributes: { name: string, value: string }[] = []
@@ -135,6 +142,10 @@ export class PeertubePlayerManager {
           p2pEnabled: options.common.p2pEnabled
         })
 
+        if (options.common.storyboard) {
+          player.storyboard(options.common.storyboard)
+        }
+
         player.on('p2pInfo', (_, data: PlayerNetworkInfo) => {
           if (data.source !== 'p2p-media-loader' || isNaN(data.bandwidthEstimate)) return
 
diff --git a/client/src/assets/player/shared/control-bar/index.ts b/client/src/assets/player/shared/control-bar/index.ts
index e71e90713..24877c267 100644
--- a/client/src/assets/player/shared/control-bar/index.ts
+++ b/client/src/assets/player/shared/control-bar/index.ts
@@ -3,4 +3,5 @@ export * from './p2p-info-button'
 export * from './peertube-link-button'
 export * from './peertube-live-display'
 export * from './peertube-load-progress-bar'
+export * from './storyboard-plugin'
 export * from './theater-button'
diff --git a/client/src/assets/player/shared/control-bar/storyboard-plugin.ts b/client/src/assets/player/shared/control-bar/storyboard-plugin.ts
new file mode 100644
index 000000000..c1843f595
--- /dev/null
+++ b/client/src/assets/player/shared/control-bar/storyboard-plugin.ts
@@ -0,0 +1,184 @@
+import videojs from 'video.js'
+import { StoryboardOptions } from '../../types'
+
+// Big thanks to this beautiful plugin: https://github.com/phloxic/videojs-sprite-thumbnails
+// Adapted to respect peertube player style
+
+const Plugin = videojs.getPlugin('plugin')
+
+class StoryboardPlugin extends Plugin {
+  private url: string
+  private height: number
+  private width: number
+  private interval: number
+
+  private cached: boolean
+
+  private mouseTimeTooltip: videojs.MouseTimeDisplay
+  private seekBar: { el(): HTMLElement, mouseTimeDisplay: any, playProgressBar: any }
+  private progress: any
+
+  private spritePlaceholder: HTMLElement
+
+  private readonly sprites: { [id: string]: HTMLImageElement } = {}
+
+  private readonly boundedHijackMouseTooltip: typeof StoryboardPlugin.prototype.hijackMouseTooltip
+
+  constructor (player: videojs.Player, options: videojs.ComponentOptions & StoryboardOptions) {
+    super(player, options)
+
+    this.url = options.url
+    this.height = options.height
+    this.width = options.width
+    this.interval = options.interval
+
+    this.boundedHijackMouseTooltip = this.hijackMouseTooltip.bind(this)
+
+    this.player.ready(() => {
+      player.addClass('vjs-storyboard')
+
+      this.init()
+    })
+  }
+
+  init () {
+    const controls = this.player.controlBar as any
+
+    // default control bar component tree is expected
+    // https://docs.videojs.com/tutorial-components.html#default-component-tree
+    this.progress = controls?.progressControl
+    this.seekBar = this.progress?.seekBar
+
+    this.mouseTimeTooltip = this.seekBar?.mouseTimeDisplay?.timeTooltip
+
+    this.spritePlaceholder = videojs.dom.createEl('div', { className: 'vjs-storyboard-sprite-placeholder' }) as HTMLElement
+    this.seekBar?.el()?.appendChild(this.spritePlaceholder)
+
+    this.player.on([ 'ready', 'loadstart' ], evt => {
+      if (evt !== 'ready') {
+        const spriteSource = this.player.currentSources().find(source => {
+          return Object.prototype.hasOwnProperty.call(source, 'storyboard')
+        }) as any
+        const spriteOpts = spriteSource?.['storyboard'] as StoryboardOptions
+
+        if (spriteOpts) {
+          this.url = spriteOpts.url
+          this.height = spriteOpts.height
+          this.width = spriteOpts.width
+          this.interval = spriteOpts.interval
+        }
+      }
+
+      this.cached = !!this.sprites[this.url]
+
+      this.load()
+    })
+  }
+
+  private load () {
+    const spriteEvents = [ 'mousemove', 'touchmove' ]
+
+    if (this.isReady()) {
+      if (!this.cached) {
+        this.sprites[this.url] = videojs.dom.createEl('img', {
+          src: this.url
+        })
+      }
+      this.progress.on(spriteEvents, this.boundedHijackMouseTooltip)
+    } else {
+      this.progress.off(spriteEvents, this.boundedHijackMouseTooltip)
+
+      this.resetMouseTooltip()
+    }
+  }
+
+  private hijackMouseTooltip (evt: Event) {
+    const sprite = this.sprites[this.url]
+    const imgWidth = sprite.naturalWidth
+    const imgHeight = sprite.naturalHeight
+    const seekBarEl = this.seekBar.el()
+
+    if (!sprite.complete || !imgWidth || !imgHeight) {
+      this.resetMouseTooltip()
+      return
+    }
+
+    this.player.requestNamedAnimationFrame('StoryBoardPlugin#hijackMouseTooltip', () => {
+      const seekBarRect = videojs.dom.getBoundingClientRect(seekBarEl)
+      const playerRect = videojs.dom.getBoundingClientRect(this.player.el())
+
+      if (!seekBarRect || !playerRect) return
+
+      const seekBarX = videojs.dom.getPointerPosition(seekBarEl, evt).x
+      let position = seekBarX * this.player.duration()
+
+      const maxPosition = Math.round((imgHeight / this.height) * (imgWidth / this.width)) - 1
+      position = Math.min(position / this.interval, maxPosition)
+
+      const responsive = 600
+      const playerWidth = this.player.currentWidth()
+      const scaleFactor = responsive && playerWidth < responsive
+        ? playerWidth / responsive
+        : 1
+      const columns = imgWidth / this.width
+
+      const scaledWidth = this.width * scaleFactor
+      const scaledHeight = this.height * scaleFactor
+      const cleft = Math.floor(position % columns) * -scaledWidth
+      const ctop = Math.floor(position / columns) * -scaledHeight
+
+      const bgSize = `${imgWidth * scaleFactor}px ${imgHeight * scaleFactor}px`
+      const topOffset = -scaledHeight - 60
+
+      const previewHalfSize = Math.round(scaledWidth / 2)
+      let left = seekBarRect.width * seekBarX - previewHalfSize
+
+      // Seek bar doesn't take all the player width, so we can add/minus a few more pixels
+      const minLeft = playerRect.left - seekBarRect.left
+      const maxLeft = seekBarRect.width - scaledWidth + (playerRect.right - seekBarRect.right)
+
+      if (left < minLeft) left = minLeft
+      if (left > maxLeft) left = maxLeft
+
+      const tooltipStyle: { [id: string]: string } = {
+        'background-image': `url("${this.url}")`,
+        'background-repeat': 'no-repeat',
+        'background-position': `${cleft}px ${ctop}px`,
+        'background-size': bgSize,
+
+        'color': '#fff',
+        'text-shadow': '1px 1px #000',
+
+        'position': 'relative',
+
+        'top': `${topOffset}px`,
+
+        'border': '1px solid #000',
+
+        // border should not overlay thumbnail area
+        'width': `${scaledWidth + 2}px`,
+        'height': `${scaledHeight + 2}px`
+      }
+
+      tooltipStyle.left = `${left}px`
+
+      for (const [ key, value ] of Object.entries(tooltipStyle)) {
+        this.spritePlaceholder.style.setProperty(key, value)
+      }
+    })
+  }
+
+  private resetMouseTooltip () {
+    if (this.spritePlaceholder) {
+      this.spritePlaceholder.style.cssText = ''
+    }
+  }
+
+  private isReady () {
+    return this.mouseTimeTooltip && this.width && this.height && this.url
+  }
+}
+
+videojs.registerPlugin('storyboard', StoryboardPlugin)
+
+export { StoryboardPlugin }
diff --git a/client/src/assets/player/types/manager-options.ts b/client/src/assets/player/types/manager-options.ts
index 1f3a0aa2e..a73341b4c 100644
--- a/client/src/assets/player/types/manager-options.ts
+++ b/client/src/assets/player/types/manager-options.ts
@@ -1,6 +1,6 @@
 import { PluginsManager } from '@root-helpers/plugins-manager'
 import { LiveVideoLatencyMode, VideoFile } from '@shared/models'
-import { PlaylistPluginOptions, VideoJSCaption } from './peertube-videojs-typings'
+import { PlaylistPluginOptions, VideoJSCaption, VideoJSStoryboard } from './peertube-videojs-typings'
 
 export type PlayerMode = 'webtorrent' | 'p2p-media-loader'
 
@@ -78,6 +78,7 @@ export interface CommonOptions extends CustomizationOptions {
   language?: string
 
   videoCaptions: VideoJSCaption[]
+  storyboard: VideoJSStoryboard
 
   videoUUID: string
   videoShortUUID: string
diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts
index 723c42c5d..30d2b287f 100644
--- a/client/src/assets/player/types/peertube-videojs-typings.ts
+++ b/client/src/assets/player/types/peertube-videojs-typings.ts
@@ -49,6 +49,8 @@ declare module 'video.js' {
 
     stats (options?: StatsCardOptions): StatsForNerdsPlugin
 
+    storyboard (options: StoryboardOptions): void
+
     textTracks (): TextTrackList & {
       tracks_: (TextTrack & { id: string, label: string, src: string })[]
     }
@@ -89,6 +91,13 @@ type VideoJSCaption = {
   src: string
 }
 
+type VideoJSStoryboard = {
+  url: string
+  width: number
+  height: number
+  interval: number
+}
+
 type PeerTubePluginOptions = {
   mode: PlayerMode
 
@@ -118,6 +127,13 @@ type MetricsPluginOptions = {
   videoUUID: string
 }
 
+type StoryboardOptions = {
+  url: string
+  width: number
+  height: number
+  interval: number
+}
+
 type PlaylistPluginOptions = {
   elements: VideoPlaylistElement[]
 
@@ -238,6 +254,7 @@ type PlaylistItemOptions = {
 
 export {
   PlayerNetworkInfo,
+  VideoJSStoryboard,
   PlaylistItemOptions,
   NextPreviousVideoButtonOptions,
   ResolutionUpdateData,
@@ -251,6 +268,7 @@ export {
   PeerTubeResolution,
   VideoJSPluginOptions,
   LoadedQualityData,
+  StoryboardOptions,
   PeerTubeLinkButtonOptions,
   PeerTubeP2PInfoButtonOptions
 }