aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2023-06-01 14:51:16 +0200
committerChocobozzz <me@florianbigard.com>2023-06-29 10:16:55 +0200
commitd8f39b126d9fe4bec1c12fb213548cc6edc87867 (patch)
tree7f0f1cb23165cf4dd789b2d78b1fef7ee116f647
parent1fb7d094229acdc190c3f7551b43ac5445814dee (diff)
downloadPeerTube-d8f39b126d9fe4bec1c12fb213548cc6edc87867.tar.gz
PeerTube-d8f39b126d9fe4bec1c12fb213548cc6edc87867.tar.zst
PeerTube-d8f39b126d9fe4bec1c12fb213548cc6edc87867.zip
Add storyboard support
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html14
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts2
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts12
-rw-r--r--client/src/app/+videos/+video-watch/video-watch.component.ts45
-rw-r--r--client/src/app/shared/form-validators/custom-config-validators.ts17
-rw-r--r--client/src/app/shared/shared-main/video/video.service.ts24
-rw-r--r--client/src/assets/player/peertube-player-manager.ts11
-rw-r--r--client/src/assets/player/shared/control-bar/index.ts1
-rw-r--r--client/src/assets/player/shared/control-bar/storyboard-plugin.ts184
-rw-r--r--client/src/assets/player/types/manager-options.ts3
-rw-r--r--client/src/assets/player/types/peertube-videojs-typings.ts18
-rw-r--r--client/src/sass/player/control-bar.scss15
-rw-r--r--client/src/sass/player/mobile.scss25
-rw-r--r--config/default.yaml3
-rw-r--r--config/production.yaml.example3
-rw-r--r--config/test-1.yaml1
-rw-r--r--config/test-2.yaml1
-rw-r--r--config/test-3.yaml1
-rw-r--r--config/test-4.yaml1
-rw-r--r--config/test-5.yaml1
-rw-r--r--config/test-6.yaml1
-rw-r--r--config/test.yaml2
-rw-r--r--server.ts3
-rw-r--r--server/controllers/activitypub/client.ts10
-rw-r--r--server/controllers/api/config.ts3
-rw-r--r--server/controllers/api/videos/index.ts2
-rw-r--r--server/controllers/api/videos/storyboard.ts29
-rw-r--r--server/controllers/api/videos/upload.ts9
-rw-r--r--server/controllers/lazy-static.ts15
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts39
-rw-r--r--server/initializers/checker-before-init.ts3
-rw-r--r--server/initializers/config.ts4
-rw-r--r--server/initializers/constants.ts19
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/lib/activitypub/context.ts13
-rw-r--r--server/lib/activitypub/send/send-update.ts14
-rw-r--r--server/lib/activitypub/videos/federate.ts13
-rw-r--r--server/lib/activitypub/videos/shared/abstract-builder.ts12
-rw-r--r--server/lib/activitypub/videos/shared/creator.ts1
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts26
-rw-r--r--server/lib/activitypub/videos/updater.ts5
-rw-r--r--server/lib/files-cache/index.ts3
-rw-r--r--server/lib/files-cache/videos-storyboard-cache.ts53
-rw-r--r--server/lib/job-queue/handlers/generate-storyboard.ts138
-rw-r--r--server/lib/job-queue/handlers/video-import.ts9
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts11
-rw-r--r--server/lib/redis.ts4
-rw-r--r--server/lib/transcoding/web-transcoding.ts18
-rw-r--r--server/middlewares/validators/config.ts1
-rw-r--r--server/models/video/formatter/video-format-utils.ts60
-rw-r--r--server/models/video/storyboard.ts169
-rw-r--r--server/models/video/video-caption.ts8
-rw-r--r--server/models/video/video.ts45
-rw-r--r--server/tests/api/check-params/config.ts3
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/video-storyboards.ts45
-rw-r--r--server/tests/api/check-params/videos-overviews.ts2
-rw-r--r--server/tests/api/server/config.ts5
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/video-storyboard.ts184
-rw-r--r--server/tests/fixtures/video_very_long_10p.mp4bin0 -> 185338 bytes
-rw-r--r--server/types/models/video/index.ts1
-rw-r--r--server/types/models/video/storyboard.ts15
-rw-r--r--server/types/models/video/video-caption.ts2
-rw-r--r--server/types/models/video/video.ts8
-rw-r--r--shared/ffmpeg/ffmpeg-images.ts37
-rw-r--r--shared/models/activitypub/objects/index.ts2
-rw-r--r--shared/models/activitypub/objects/video-object.ts (renamed from shared/models/activitypub/objects/video-torrent-object.ts)16
-rw-r--r--shared/models/server/custom-config.model.ts4
-rw-r--r--shared/models/server/job.model.ts8
-rw-r--r--shared/models/videos/index.ts1
-rw-r--r--shared/models/videos/storyboard.model.ts11
-rw-r--r--shared/server-commands/server/config-command.ts7
-rw-r--r--shared/server-commands/server/jobs.ts15
-rw-r--r--shared/server-commands/server/server.ts5
-rw-r--r--shared/server-commands/videos/index.ts1
-rw-r--r--shared/server-commands/videos/storyboard-command.ts19
-rw-r--r--support/doc/api/openapi.yaml35
79 files changed, 1476 insertions, 100 deletions
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html
index bbf946df0..9701e7f85 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html
+++ b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html
@@ -52,6 +52,20 @@
52 52
53 <div *ngIf="formErrors.cache.torrents.size" class="form-error">{{ formErrors.cache.torrents.size }}</div> 53 <div *ngIf="formErrors.cache.torrents.size" class="form-error">{{ formErrors.cache.torrents.size }}</div>
54 </div> 54 </div>
55
56 <div class="form-group" formGroupName="torrents">
57 <label i18n for="cacheTorrentsSize">Number of video storyboard images to keep in cache</label>
58
59 <div class="number-with-unit">
60 <input
61 type="number" min="0" id="cacheStoryboardsSize" class="form-control"
62 formControlName="size" [ngClass]="{ 'input-error': formErrors['cache.storyboards.size'] }"
63 >
64 <span i18n>{getCacheSize('storyboards'), plural, =1 {cached storyboard} other {cached storyboards}}</span>
65 </div>
66
67 <div *ngIf="formErrors.cache.storyboards.size" class="form-error">{{ formErrors.cache.storyboards.size }}</div>
68 </div>
55 </ng-container> 69 </ng-container>
56 70
57 </div> 71 </div>
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts
index 79a98f288..06c5e6221 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts
+++ b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts
@@ -10,7 +10,7 @@ export class EditAdvancedConfigurationComponent {
10 @Input() form: FormGroup 10 @Input() form: FormGroup
11 @Input() formErrors: any 11 @Input() formErrors: any
12 12
13 getCacheSize (type: 'captions' | 'previews' | 'torrents') { 13 getCacheSize (type: 'captions' | 'previews' | 'torrents' | 'storyboards') {
14 return this.form.value['cache'][type]['size'] 14 return this.form.value['cache'][type]['size']
15 } 15 }
16} 16}
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
index 2c3b7560d..9219d608b 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
+++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
@@ -9,8 +9,7 @@ import { Notifier } from '@app/core'
9import { ServerService } from '@app/core/server/server.service' 9import { ServerService } from '@app/core/server/server.service'
10import { 10import {
11 ADMIN_EMAIL_VALIDATOR, 11 ADMIN_EMAIL_VALIDATOR,
12 CACHE_CAPTIONS_SIZE_VALIDATOR, 12 CACHE_SIZE_VALIDATOR,
13 CACHE_PREVIEWS_SIZE_VALIDATOR,
14 CONCURRENCY_VALIDATOR, 13 CONCURRENCY_VALIDATOR,
15 INDEX_URL_VALIDATOR, 14 INDEX_URL_VALIDATOR,
16 INSTANCE_NAME_VALIDATOR, 15 INSTANCE_NAME_VALIDATOR,
@@ -120,13 +119,16 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
120 }, 119 },
121 cache: { 120 cache: {
122 previews: { 121 previews: {
123 size: CACHE_PREVIEWS_SIZE_VALIDATOR 122 size: CACHE_SIZE_VALIDATOR
124 }, 123 },
125 captions: { 124 captions: {
126 size: CACHE_CAPTIONS_SIZE_VALIDATOR 125 size: CACHE_SIZE_VALIDATOR
127 }, 126 },
128 torrents: { 127 torrents: {
129 size: CACHE_CAPTIONS_SIZE_VALIDATOR 128 size: CACHE_SIZE_VALIDATOR
129 },
130 storyboards: {
131 size: CACHE_SIZE_VALIDATOR
130 } 132 }
131 }, 133 },
132 signup: { 134 signup: {
diff --git a/client/src/app/+videos/+video-watch/video-watch.component.ts b/client/src/app/+videos/+video-watch/video-watch.component.ts
index aba3ee086..43744789d 100644
--- a/client/src/app/+videos/+video-watch/video-watch.component.ts
+++ b/client/src/app/+videos/+video-watch/video-watch.component.ts
@@ -33,6 +33,7 @@ import {
33 LiveVideo, 33 LiveVideo,
34 PeerTubeProblemDocument, 34 PeerTubeProblemDocument,
35 ServerErrorCode, 35 ServerErrorCode,
36 Storyboard,
36 VideoCaption, 37 VideoCaption,
37 VideoPrivacy, 38 VideoPrivacy,
38 VideoState 39 VideoState
@@ -69,6 +70,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
69 videoCaptions: VideoCaption[] = [] 70 videoCaptions: VideoCaption[] = []
70 liveVideo: LiveVideo 71 liveVideo: LiveVideo
71 videoPassword: string 72 videoPassword: string
73 storyboards: Storyboard[] = []
72 74
73 playlistPosition: number 75 playlistPosition: number
74 playlist: VideoPlaylist = null 76 playlist: VideoPlaylist = null
@@ -285,9 +287,10 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
285 forkJoin([ 287 forkJoin([
286 videoAndLiveObs, 288 videoAndLiveObs,
287 this.videoCaptionService.listCaptions(videoId, videoPassword), 289 this.videoCaptionService.listCaptions(videoId, videoPassword),
290 this.videoService.getStoryboards(videoId),
288 this.userService.getAnonymousOrLoggedUser() 291 this.userService.getAnonymousOrLoggedUser()
289 ]).subscribe({ 292 ]).subscribe({
290 next: ([ { video, live, videoFileToken }, captionsResult, loggedInOrAnonymousUser ]) => { 293 next: ([ { video, live, videoFileToken }, captionsResult, storyboards, loggedInOrAnonymousUser ]) => {
291 const queryParams = this.route.snapshot.queryParams 294 const queryParams = this.route.snapshot.queryParams
292 295
293 const urlOptions = { 296 const urlOptions = {
@@ -309,6 +312,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
309 video, 312 video,
310 live, 313 live,
311 videoCaptions: captionsResult.data, 314 videoCaptions: captionsResult.data,
315 storyboards,
312 videoFileToken, 316 videoFileToken,
313 videoPassword, 317 videoPassword,
314 loggedInOrAnonymousUser, 318 loggedInOrAnonymousUser,
@@ -414,6 +418,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
414 video: VideoDetails 418 video: VideoDetails
415 live: LiveVideo 419 live: LiveVideo
416 videoCaptions: VideoCaption[] 420 videoCaptions: VideoCaption[]
421 storyboards: Storyboard[]
417 videoFileToken: string 422 videoFileToken: string
418 videoPassword: string 423 videoPassword: string
419 424
@@ -421,7 +426,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
421 loggedInOrAnonymousUser: User 426 loggedInOrAnonymousUser: User
422 forceAutoplay: boolean 427 forceAutoplay: boolean
423 }) { 428 }) {
424 const { video, live, videoCaptions, urlOptions, videoFileToken, videoPassword, loggedInOrAnonymousUser, forceAutoplay } = options 429 const {
430 video,
431 live,
432 videoCaptions,
433 storyboards,
434 urlOptions,
435 videoFileToken,
436 videoPassword,
437 loggedInOrAnonymousUser,
438 forceAutoplay
439 } = options
425 440
426 this.subscribeToLiveEventsIfNeeded(this.video, video) 441 this.subscribeToLiveEventsIfNeeded(this.video, video)
427 442
@@ -430,6 +445,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
430 this.liveVideo = live 445 this.liveVideo = live
431 this.videoFileToken = videoFileToken 446 this.videoFileToken = videoFileToken
432 this.videoPassword = videoPassword 447 this.videoPassword = videoPassword
448 this.storyboards = storyboards
433 449
434 // Re init attributes 450 // Re init attributes
435 this.playerPlaceholderImgSrc = undefined 451 this.playerPlaceholderImgSrc = undefined
@@ -485,6 +501,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
485 const params = { 501 const params = {
486 video: this.video, 502 video: this.video,
487 videoCaptions: this.videoCaptions, 503 videoCaptions: this.videoCaptions,
504 storyboards: this.storyboards,
488 liveVideo: this.liveVideo, 505 liveVideo: this.liveVideo,
489 videoFileToken: this.videoFileToken, 506 videoFileToken: this.videoFileToken,
490 videoPassword: this.videoPassword, 507 videoPassword: this.videoPassword,
@@ -636,6 +653,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
636 video: VideoDetails 653 video: VideoDetails
637 liveVideo: LiveVideo 654 liveVideo: LiveVideo
638 videoCaptions: VideoCaption[] 655 videoCaptions: VideoCaption[]
656 storyboards: Storyboard[]
639 657
640 videoFileToken: string 658 videoFileToken: string
641 videoPassword: string 659 videoPassword: string
@@ -646,7 +664,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
646 forceAutoplay: boolean 664 forceAutoplay: boolean
647 user?: AuthUser // Keep for plugins 665 user?: AuthUser // Keep for plugins
648 }) { 666 }) {
649 const { video, liveVideo, videoCaptions, videoFileToken, videoPassword, urlOptions, loggedInOrAnonymousUser, forceAutoplay } = params 667 const {
668 video,
669 liveVideo,
670 videoCaptions,
671 storyboards,
672 videoFileToken,
673 videoPassword,
674 urlOptions,
675 loggedInOrAnonymousUser,
676 forceAutoplay
677 } = params
650 678
651 const getStartTime = () => { 679 const getStartTime = () => {
652 const byUrl = urlOptions.startTime !== undefined 680 const byUrl = urlOptions.startTime !== undefined
@@ -673,6 +701,15 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
673 src: environment.apiUrl + c.captionPath 701 src: environment.apiUrl + c.captionPath
674 })) 702 }))
675 703
704 const storyboard = storyboards.length !== 0
705 ? {
706 url: environment.apiUrl + storyboards[0].storyboardPath,
707 height: storyboards[0].spriteHeight,
708 width: storyboards[0].spriteWidth,
709 interval: storyboards[0].spriteDuration
710 }
711 : undefined
712
676 const liveOptions = video.isLive 713 const liveOptions = video.isLive
677 ? { latencyMode: liveVideo.latencyMode } 714 ? { latencyMode: liveVideo.latencyMode }
678 : undefined 715 : undefined
@@ -734,6 +771,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
734 videoPassword: () => videoPassword, 771 videoPassword: () => videoPassword,
735 772
736 videoCaptions: playerCaptions, 773 videoCaptions: playerCaptions,
774 storyboard,
737 775
738 videoShortUUID: video.shortUUID, 776 videoShortUUID: video.shortUUID,
739 videoUUID: video.uuid, 777 videoUUID: video.uuid,
@@ -767,6 +805,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
767 else mode = 'webtorrent' 805 else mode = 'webtorrent'
768 } 806 }
769 807
808 // FIXME: remove, we don't support these old web browsers anymore
770 // p2p-media-loader needs TextEncoder, fallback on WebTorrent if not available 809 // p2p-media-loader needs TextEncoder, fallback on WebTorrent if not available
771 if (typeof TextEncoder === 'undefined') { 810 if (typeof TextEncoder === 'undefined') {
772 mode = 'webtorrent' 811 mode = 'webtorrent'
diff --git a/client/src/app/shared/form-validators/custom-config-validators.ts b/client/src/app/shared/form-validators/custom-config-validators.ts
index ff0813f7d..3672e5610 100644
--- a/client/src/app/shared/form-validators/custom-config-validators.ts
+++ b/client/src/app/shared/form-validators/custom-config-validators.ts
@@ -22,21 +22,12 @@ export const SERVICES_TWITTER_USERNAME_VALIDATOR: BuildFormValidator = {
22 } 22 }
23} 23}
24 24
25export const CACHE_PREVIEWS_SIZE_VALIDATOR: BuildFormValidator = { 25export const CACHE_SIZE_VALIDATOR: BuildFormValidator = {
26 VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ], 26 VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ],
27 MESSAGES: { 27 MESSAGES: {
28 required: $localize`Previews cache size is required.`, 28 required: $localize`Cache size is required.`,
29 min: $localize`Previews cache size must be greater than 1.`, 29 min: $localize`Cache size must be greater than 1.`,
30 pattern: $localize`Previews cache size must be a number.` 30 pattern: $localize`Cache size must be a number.`
31 }
32}
33
34export const CACHE_CAPTIONS_SIZE_VALIDATOR: BuildFormValidator = {
35 VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ],
36 MESSAGES: {
37 required: $localize`Captions cache size is required.`,
38 min: $localize`Captions cache size must be greater than 1.`,
39 pattern: $localize`Captions cache size must be a number.`
40 } 31 }
41} 32}
42 33
diff --git a/client/src/app/shared/shared-main/video/video.service.ts b/client/src/app/shared/shared-main/video/video.service.ts
index d67a2e192..c2e3d7511 100644
--- a/client/src/app/shared/shared-main/video/video.service.ts
+++ b/client/src/app/shared/shared-main/video/video.service.ts
@@ -11,6 +11,7 @@ import {
11 FeedFormat, 11 FeedFormat,
12 NSFWPolicyType, 12 NSFWPolicyType,
13 ResultList, 13 ResultList,
14 Storyboard,
14 UserVideoRate, 15 UserVideoRate,
15 UserVideoRateType, 16 UserVideoRateType,
16 UserVideoRateUpdate, 17 UserVideoRateUpdate,
@@ -344,6 +345,25 @@ export class VideoService {
344 ) 345 )
345 } 346 }
346 347
348 // ---------------------------------------------------------------------------
349
350 getStoryboards (videoId: string | number) {
351 return this.authHttp
352 .get<{ storyboards: Storyboard[] }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/storyboards')
353 .pipe(
354 map(({ storyboards }) => storyboards),
355 catchError(err => {
356 if (err.status === 404) {
357 return of([])
358 }
359
360 this.restExtractor.handleError(err)
361 })
362 )
363 }
364
365 // ---------------------------------------------------------------------------
366
347 getSource (videoId: number) { 367 getSource (videoId: number) {
348 return this.authHttp 368 return this.authHttp
349 .get<{ source: VideoSource }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source') 369 .get<{ source: VideoSource }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source')
@@ -358,6 +378,8 @@ export class VideoService {
358 ) 378 )
359 } 379 }
360 380
381 // ---------------------------------------------------------------------------
382
361 setVideoLike (id: string, videoPassword: string) { 383 setVideoLike (id: string, videoPassword: string) {
362 return this.setVideoRate(id, 'like', videoPassword) 384 return this.setVideoRate(id, 'like', videoPassword)
363 } 385 }
@@ -370,6 +392,8 @@ export class VideoService {
370 return this.setVideoRate(id, 'none', videoPassword) 392 return this.setVideoRate(id, 'none', videoPassword)
371 } 393 }
372 394
395 // ---------------------------------------------------------------------------
396
373 getUserVideoRating (id: string) { 397 getUserVideoRating (id: string) {
374 const url = UserService.BASE_USERS_URL + 'me/videos/' + id + '/rating' 398 const url = UserService.BASE_USERS_URL + 'me/videos/' + id + '/rating'
375 399
diff --git a/client/src/assets/player/peertube-player-manager.ts b/client/src/assets/player/peertube-player-manager.ts
index 2781850b9..66d9c7298 100644
--- a/client/src/assets/player/peertube-player-manager.ts
+++ b/client/src/assets/player/peertube-player-manager.ts
@@ -6,6 +6,7 @@ import './shared/stats/stats-plugin'
6import './shared/bezels/bezels-plugin' 6import './shared/bezels/bezels-plugin'
7import './shared/peertube/peertube-plugin' 7import './shared/peertube/peertube-plugin'
8import './shared/resolutions/peertube-resolutions-plugin' 8import './shared/resolutions/peertube-resolutions-plugin'
9import './shared/control-bar/storyboard-plugin'
9import './shared/control-bar/next-previous-video-button' 10import './shared/control-bar/next-previous-video-button'
10import './shared/control-bar/p2p-info-button' 11import './shared/control-bar/p2p-info-button'
11import './shared/control-bar/peertube-link-button' 12import './shared/control-bar/peertube-link-button'
@@ -42,6 +43,12 @@ CaptionsButton.prototype.controlText_ = 'Subtitles/CC'
42// We just want to display 'Off' instead of 'captions off', keep a space so the variable == true (hacky I know) 43// We just want to display 'Off' instead of 'captions off', keep a space so the variable == true (hacky I know)
43CaptionsButton.prototype.label_ = ' ' 44CaptionsButton.prototype.label_ = ' '
44 45
46// TODO: remove when https://github.com/videojs/video.js/pull/7598 is merged
47const PlayProgressBar = videojs.getComponent('PlayProgressBar') as any
48if (PlayProgressBar.prototype.options_.children.includes('timeTooltip') !== true) {
49 PlayProgressBar.prototype.options_.children.push('timeTooltip')
50}
51
45export class PeertubePlayerManager { 52export class PeertubePlayerManager {
46 private static playerElementClassName: string 53 private static playerElementClassName: string
47 private static playerElementAttributes: { name: string, value: string }[] = [] 54 private static playerElementAttributes: { name: string, value: string }[] = []
@@ -135,6 +142,10 @@ export class PeertubePlayerManager {
135 p2pEnabled: options.common.p2pEnabled 142 p2pEnabled: options.common.p2pEnabled
136 }) 143 })
137 144
145 if (options.common.storyboard) {
146 player.storyboard(options.common.storyboard)
147 }
148
138 player.on('p2pInfo', (_, data: PlayerNetworkInfo) => { 149 player.on('p2pInfo', (_, data: PlayerNetworkInfo) => {
139 if (data.source !== 'p2p-media-loader' || isNaN(data.bandwidthEstimate)) return 150 if (data.source !== 'p2p-media-loader' || isNaN(data.bandwidthEstimate)) return
140 151
diff --git a/client/src/assets/player/shared/control-bar/index.ts b/client/src/assets/player/shared/control-bar/index.ts
index e71e90713..24877c267 100644
--- a/client/src/assets/player/shared/control-bar/index.ts
+++ b/client/src/assets/player/shared/control-bar/index.ts
@@ -3,4 +3,5 @@ export * from './p2p-info-button'
3export * from './peertube-link-button' 3export * from './peertube-link-button'
4export * from './peertube-live-display' 4export * from './peertube-live-display'
5export * from './peertube-load-progress-bar' 5export * from './peertube-load-progress-bar'
6export * from './storyboard-plugin'
6export * from './theater-button' 7export * from './theater-button'
diff --git a/client/src/assets/player/shared/control-bar/storyboard-plugin.ts b/client/src/assets/player/shared/control-bar/storyboard-plugin.ts
new file mode 100644
index 000000000..c1843f595
--- /dev/null
+++ b/client/src/assets/player/shared/control-bar/storyboard-plugin.ts
@@ -0,0 +1,184 @@
1import videojs from 'video.js'
2import { StoryboardOptions } from '../../types'
3
4// Big thanks to this beautiful plugin: https://github.com/phloxic/videojs-sprite-thumbnails
5// Adapted to respect peertube player style
6
7const Plugin = videojs.getPlugin('plugin')
8
9class StoryboardPlugin extends Plugin {
10 private url: string
11 private height: number
12 private width: number
13 private interval: number
14
15 private cached: boolean
16
17 private mouseTimeTooltip: videojs.MouseTimeDisplay
18 private seekBar: { el(): HTMLElement, mouseTimeDisplay: any, playProgressBar: any }
19 private progress: any
20
21 private spritePlaceholder: HTMLElement
22
23 private readonly sprites: { [id: string]: HTMLImageElement } = {}
24
25 private readonly boundedHijackMouseTooltip: typeof StoryboardPlugin.prototype.hijackMouseTooltip
26
27 constructor (player: videojs.Player, options: videojs.ComponentOptions & StoryboardOptions) {
28 super(player, options)
29
30 this.url = options.url
31 this.height = options.height
32 this.width = options.width
33 this.interval = options.interval
34
35 this.boundedHijackMouseTooltip = this.hijackMouseTooltip.bind(this)
36
37 this.player.ready(() => {
38 player.addClass('vjs-storyboard')
39
40 this.init()
41 })
42 }
43
44 init () {
45 const controls = this.player.controlBar as any
46
47 // default control bar component tree is expected
48 // https://docs.videojs.com/tutorial-components.html#default-component-tree
49 this.progress = controls?.progressControl
50 this.seekBar = this.progress?.seekBar
51
52 this.mouseTimeTooltip = this.seekBar?.mouseTimeDisplay?.timeTooltip
53
54 this.spritePlaceholder = videojs.dom.createEl('div', { className: 'vjs-storyboard-sprite-placeholder' }) as HTMLElement
55 this.seekBar?.el()?.appendChild(this.spritePlaceholder)
56
57 this.player.on([ 'ready', 'loadstart' ], evt => {
58 if (evt !== 'ready') {
59 const spriteSource = this.player.currentSources().find(source => {
60 return Object.prototype.hasOwnProperty.call(source, 'storyboard')
61 }) as any
62 const spriteOpts = spriteSource?.['storyboard'] as StoryboardOptions
63
64 if (spriteOpts) {
65 this.url = spriteOpts.url
66 this.height = spriteOpts.height
67 this.width = spriteOpts.width
68 this.interval = spriteOpts.interval
69 }
70 }
71
72 this.cached = !!this.sprites[this.url]
73
74 this.load()
75 })
76 }
77
78 private load () {
79 const spriteEvents = [ 'mousemove', 'touchmove' ]
80
81 if (this.isReady()) {
82 if (!this.cached) {
83 this.sprites[this.url] = videojs.dom.createEl('img', {
84 src: this.url
85 })
86 }
87 this.progress.on(spriteEvents, this.boundedHijackMouseTooltip)
88 } else {
89 this.progress.off(spriteEvents, this.boundedHijackMouseTooltip)
90
91 this.resetMouseTooltip()
92 }
93 }
94
95 private hijackMouseTooltip (evt: Event) {
96 const sprite = this.sprites[this.url]
97 const imgWidth = sprite.naturalWidth
98 const imgHeight = sprite.naturalHeight
99 const seekBarEl = this.seekBar.el()
100
101 if (!sprite.complete || !imgWidth || !imgHeight) {
102 this.resetMouseTooltip()
103 return
104 }
105
106 this.player.requestNamedAnimationFrame('StoryBoardPlugin#hijackMouseTooltip', () => {
107 const seekBarRect = videojs.dom.getBoundingClientRect(seekBarEl)
108 const playerRect = videojs.dom.getBoundingClientRect(this.player.el())
109
110 if (!seekBarRect || !playerRect) return
111
112 const seekBarX = videojs.dom.getPointerPosition(seekBarEl, evt).x
113 let position = seekBarX * this.player.duration()
114
115 const maxPosition = Math.round((imgHeight / this.height) * (imgWidth / this.width)) - 1
116 position = Math.min(position / this.interval, maxPosition)
117
118 const responsive = 600
119 const playerWidth = this.player.currentWidth()
120 const scaleFactor = responsive && playerWidth < responsive
121 ? playerWidth / responsive
122 : 1
123 const columns = imgWidth / this.width
124
125 const scaledWidth = this.width * scaleFactor
126 const scaledHeight = this.height * scaleFactor
127 const cleft = Math.floor(position % columns) * -scaledWidth
128 const ctop = Math.floor(position / columns) * -scaledHeight
129
130 const bgSize = `${imgWidth * scaleFactor}px ${imgHeight * scaleFactor}px`
131 const topOffset = -scaledHeight - 60
132
133 const previewHalfSize = Math.round(scaledWidth / 2)
134 let left = seekBarRect.width * seekBarX - previewHalfSize
135
136 // Seek bar doesn't take all the player width, so we can add/minus a few more pixels
137 const minLeft = playerRect.left - seekBarRect.left
138 const maxLeft = seekBarRect.width - scaledWidth + (playerRect.right - seekBarRect.right)
139
140 if (left < minLeft) left = minLeft
141 if (left > maxLeft) left = maxLeft
142
143 const tooltipStyle: { [id: string]: string } = {
144 'background-image': `url("${this.url}")`,
145 'background-repeat': 'no-repeat',
146 'background-position': `${cleft}px ${ctop}px`,
147 'background-size': bgSize,
148
149 'color': '#fff',
150 'text-shadow': '1px 1px #000',
151
152 'position': 'relative',
153
154 'top': `${topOffset}px`,
155
156 'border': '1px solid #000',
157
158 // border should not overlay thumbnail area
159 'width': `${scaledWidth + 2}px`,
160 'height': `${scaledHeight + 2}px`
161 }
162
163 tooltipStyle.left = `${left}px`
164
165 for (const [ key, value ] of Object.entries(tooltipStyle)) {
166 this.spritePlaceholder.style.setProperty(key, value)
167 }
168 })
169 }
170
171 private resetMouseTooltip () {
172 if (this.spritePlaceholder) {
173 this.spritePlaceholder.style.cssText = ''
174 }
175 }
176
177 private isReady () {
178 return this.mouseTimeTooltip && this.width && this.height && this.url
179 }
180}
181
182videojs.registerPlugin('storyboard', StoryboardPlugin)
183
184export { StoryboardPlugin }
diff --git a/client/src/assets/player/types/manager-options.ts b/client/src/assets/player/types/manager-options.ts
index 1f3a0aa2e..a73341b4c 100644
--- a/client/src/assets/player/types/manager-options.ts
+++ b/client/src/assets/player/types/manager-options.ts
@@ -1,6 +1,6 @@
1import { PluginsManager } from '@root-helpers/plugins-manager' 1import { PluginsManager } from '@root-helpers/plugins-manager'
2import { LiveVideoLatencyMode, VideoFile } from '@shared/models' 2import { LiveVideoLatencyMode, VideoFile } from '@shared/models'
3import { PlaylistPluginOptions, VideoJSCaption } from './peertube-videojs-typings' 3import { PlaylistPluginOptions, VideoJSCaption, VideoJSStoryboard } from './peertube-videojs-typings'
4 4
5export type PlayerMode = 'webtorrent' | 'p2p-media-loader' 5export type PlayerMode = 'webtorrent' | 'p2p-media-loader'
6 6
@@ -78,6 +78,7 @@ export interface CommonOptions extends CustomizationOptions {
78 language?: string 78 language?: string
79 79
80 videoCaptions: VideoJSCaption[] 80 videoCaptions: VideoJSCaption[]
81 storyboard: VideoJSStoryboard
81 82
82 videoUUID: string 83 videoUUID: string
83 videoShortUUID: string 84 videoShortUUID: string
diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts
index 723c42c5d..30d2b287f 100644
--- a/client/src/assets/player/types/peertube-videojs-typings.ts
+++ b/client/src/assets/player/types/peertube-videojs-typings.ts
@@ -49,6 +49,8 @@ declare module 'video.js' {
49 49
50 stats (options?: StatsCardOptions): StatsForNerdsPlugin 50 stats (options?: StatsCardOptions): StatsForNerdsPlugin
51 51
52 storyboard (options: StoryboardOptions): void
53
52 textTracks (): TextTrackList & { 54 textTracks (): TextTrackList & {
53 tracks_: (TextTrack & { id: string, label: string, src: string })[] 55 tracks_: (TextTrack & { id: string, label: string, src: string })[]
54 } 56 }
@@ -89,6 +91,13 @@ type VideoJSCaption = {
89 src: string 91 src: string
90} 92}
91 93
94type VideoJSStoryboard = {
95 url: string
96 width: number
97 height: number
98 interval: number
99}
100
92type PeerTubePluginOptions = { 101type PeerTubePluginOptions = {
93 mode: PlayerMode 102 mode: PlayerMode
94 103
@@ -118,6 +127,13 @@ type MetricsPluginOptions = {
118 videoUUID: string 127 videoUUID: string
119} 128}
120 129
130type StoryboardOptions = {
131 url: string
132 width: number
133 height: number
134 interval: number
135}
136
121type PlaylistPluginOptions = { 137type PlaylistPluginOptions = {
122 elements: VideoPlaylistElement[] 138 elements: VideoPlaylistElement[]
123 139
@@ -238,6 +254,7 @@ type PlaylistItemOptions = {
238 254
239export { 255export {
240 PlayerNetworkInfo, 256 PlayerNetworkInfo,
257 VideoJSStoryboard,
241 PlaylistItemOptions, 258 PlaylistItemOptions,
242 NextPreviousVideoButtonOptions, 259 NextPreviousVideoButtonOptions,
243 ResolutionUpdateData, 260 ResolutionUpdateData,
@@ -251,6 +268,7 @@ export {
251 PeerTubeResolution, 268 PeerTubeResolution,
252 VideoJSPluginOptions, 269 VideoJSPluginOptions,
253 LoadedQualityData, 270 LoadedQualityData,
271 StoryboardOptions,
254 PeerTubeLinkButtonOptions, 272 PeerTubeLinkButtonOptions,
255 PeerTubeP2PInfoButtonOptions 273 PeerTubeP2PInfoButtonOptions
256} 274}
diff --git a/client/src/sass/player/control-bar.scss b/client/src/sass/player/control-bar.scss
index 96b3adf66..02d5fa169 100644
--- a/client/src/sass/player/control-bar.scss
+++ b/client/src/sass/player/control-bar.scss
@@ -3,6 +3,20 @@
3@use '_mixins' as *; 3@use '_mixins' as *;
4@use './_player-variables' as *; 4@use './_player-variables' as *;
5 5
6// Like the time tooltip
7.video-js .vjs-progress-holder .vjs-storyboard-sprite-placeholder {
8 display: none;
9}
10
11.video-js .vjs-progress-control:hover .vjs-storyboard-sprite-placeholder,
12.video-js .vjs-progress-control:hover .vjs-progress-holder:focus .vjs-storyboard-sprite-placeholder {
13 display: block;
14
15 // Ensure that we maintain a font-size of ~10px.
16 font-size: 0.6em;
17 visibility: visible;
18}
19
6.video-js.vjs-peertube-skin .vjs-control-bar { 20.video-js.vjs-peertube-skin .vjs-control-bar {
7 z-index: 100; 21 z-index: 100;
8 22
@@ -79,6 +93,7 @@
79 top: -0.3em; 93 top: -0.3em;
80 } 94 }
81 95
96 // Only used on mobile
82 .vjs-time-tooltip { 97 .vjs-time-tooltip {
83 display: none; 98 display: none;
84 } 99 }
diff --git a/client/src/sass/player/mobile.scss b/client/src/sass/player/mobile.scss
index 84d7a00f1..d150c54ee 100644
--- a/client/src/sass/player/mobile.scss
+++ b/client/src/sass/player/mobile.scss
@@ -6,6 +6,31 @@
6/* Special mobile style */ 6/* Special mobile style */
7 7
8.video-js.vjs-peertube-skin.vjs-is-mobile { 8.video-js.vjs-peertube-skin.vjs-is-mobile {
9 // No hover means we can't display the storyboard/time tooltip on mouse hover
10 // Use the time tooltip in progress control instead
11 .vjs-mouse-display {
12 display: none !important;
13 }
14
15 .vjs-storyboard-sprite-placeholder {
16 display: none;
17 }
18
19 .vjs-progress-control .vjs-sliding {
20
21 .vjs-time-tooltip,
22 .vjs-storyboard-sprite-placeholder {
23 display: block !important;
24
25 visibility: visible !important;
26 }
27
28 .vjs-time-tooltip {
29 color: #fff;
30 background-color: rgba(0, 0, 0, 0.8);
31 }
32 }
33
9 .vjs-control-bar { 34 .vjs-control-bar {
10 .vjs-progress-control .vjs-slider .vjs-play-progress { 35 .vjs-progress-control .vjs-slider .vjs-play-progress {
11 // Always display the circle on mobile 36 // Always display the circle on mobile
diff --git a/config/default.yaml b/config/default.yaml
index 5d0eab4f5..e54c93ac5 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -136,6 +136,7 @@ storage:
136 logs: 'storage/logs/' 136 logs: 'storage/logs/'
137 previews: 'storage/previews/' 137 previews: 'storage/previews/'
138 thumbnails: 'storage/thumbnails/' 138 thumbnails: 'storage/thumbnails/'
139 storyboards: 'storage/storyboards/'
139 torrents: 'storage/torrents/' 140 torrents: 'storage/torrents/'
140 captions: 'storage/captions/' 141 captions: 'storage/captions/'
141 cache: 'storage/cache/' 142 cache: 'storage/cache/'
@@ -396,6 +397,8 @@ cache:
396 size: 500 # Max number of video captions/subtitles you want to cache 397 size: 500 # Max number of video captions/subtitles you want to cache
397 torrents: 398 torrents:
398 size: 500 # Max number of video torrents you want to cache 399 size: 500 # Max number of video torrents you want to cache
400 storyboards:
401 size: 500 # Max number of video storyboards you want to cache
399 402
400admin: 403admin:
401 # Used to generate the root user at first startup 404 # Used to generate the root user at first startup
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 5514f1af6..83ee48dae 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -134,6 +134,7 @@ storage:
134 logs: '/var/www/peertube/storage/logs/' 134 logs: '/var/www/peertube/storage/logs/'
135 previews: '/var/www/peertube/storage/previews/' 135 previews: '/var/www/peertube/storage/previews/'
136 thumbnails: '/var/www/peertube/storage/thumbnails/' 136 thumbnails: '/var/www/peertube/storage/thumbnails/'
137 storyboards: '/var/www/peertube/storage/storyboards/'
137 torrents: '/var/www/peertube/storage/torrents/' 138 torrents: '/var/www/peertube/storage/torrents/'
138 captions: '/var/www/peertube/storage/captions/' 139 captions: '/var/www/peertube/storage/captions/'
139 cache: '/var/www/peertube/storage/cache/' 140 cache: '/var/www/peertube/storage/cache/'
@@ -406,6 +407,8 @@ cache:
406 size: 500 # Max number of video captions/subtitles you want to cache 407 size: 500 # Max number of video captions/subtitles you want to cache
407 torrents: 408 torrents:
408 size: 500 # Max number of video torrents you want to cache 409 size: 500 # Max number of video torrents you want to cache
410 storyboards:
411 size: 500 # Max number of video storyboards you want to cache
409 412
410admin: 413admin:
411 # Used to generate the root user at first startup 414 # Used to generate the root user at first startup
diff --git a/config/test-1.yaml b/config/test-1.yaml
index 7b62e3d0c..45ec27e63 100644
--- a/config/test-1.yaml
+++ b/config/test-1.yaml
@@ -19,6 +19,7 @@ storage:
19 logs: 'test1/logs/' 19 logs: 'test1/logs/'
20 previews: 'test1/previews/' 20 previews: 'test1/previews/'
21 thumbnails: 'test1/thumbnails/' 21 thumbnails: 'test1/thumbnails/'
22 storyboards: 'test1/storyboards/'
22 torrents: 'test1/torrents/' 23 torrents: 'test1/torrents/'
23 captions: 'test1/captions/' 24 captions: 'test1/captions/'
24 cache: 'test1/cache/' 25 cache: 'test1/cache/'
diff --git a/config/test-2.yaml b/config/test-2.yaml
index ba36369a6..7a06e5650 100644
--- a/config/test-2.yaml
+++ b/config/test-2.yaml
@@ -19,6 +19,7 @@ storage:
19 logs: 'test2/logs/' 19 logs: 'test2/logs/'
20 previews: 'test2/previews/' 20 previews: 'test2/previews/'
21 thumbnails: 'test2/thumbnails/' 21 thumbnails: 'test2/thumbnails/'
22 storyboards: 'test2/storyboards/'
22 torrents: 'test2/torrents/' 23 torrents: 'test2/torrents/'
23 captions: 'test2/captions/' 24 captions: 'test2/captions/'
24 cache: 'test2/cache/' 25 cache: 'test2/cache/'
diff --git a/config/test-3.yaml b/config/test-3.yaml
index 6adec7953..4b1563369 100644
--- a/config/test-3.yaml
+++ b/config/test-3.yaml
@@ -19,6 +19,7 @@ storage:
19 logs: 'test3/logs/' 19 logs: 'test3/logs/'
20 previews: 'test3/previews/' 20 previews: 'test3/previews/'
21 thumbnails: 'test3/thumbnails/' 21 thumbnails: 'test3/thumbnails/'
22 storyboards: 'test3/storyboards/'
22 torrents: 'test3/torrents/' 23 torrents: 'test3/torrents/'
23 captions: 'test3/captions/' 24 captions: 'test3/captions/'
24 cache: 'test3/cache/' 25 cache: 'test3/cache/'
diff --git a/config/test-4.yaml b/config/test-4.yaml
index f042aee46..248db4db9 100644
--- a/config/test-4.yaml
+++ b/config/test-4.yaml
@@ -19,6 +19,7 @@ storage:
19 logs: 'test4/logs/' 19 logs: 'test4/logs/'
20 previews: 'test4/previews/' 20 previews: 'test4/previews/'
21 thumbnails: 'test4/thumbnails/' 21 thumbnails: 'test4/thumbnails/'
22 storyboards: 'test4/storyboards/'
22 torrents: 'test4/torrents/' 23 torrents: 'test4/torrents/'
23 captions: 'test4/captions/' 24 captions: 'test4/captions/'
24 cache: 'test4/cache/' 25 cache: 'test4/cache/'
diff --git a/config/test-5.yaml b/config/test-5.yaml
index ad90fec04..04e2cd78d 100644
--- a/config/test-5.yaml
+++ b/config/test-5.yaml
@@ -19,6 +19,7 @@ storage:
19 logs: 'test5/logs/' 19 logs: 'test5/logs/'
20 previews: 'test5/previews/' 20 previews: 'test5/previews/'
21 thumbnails: 'test5/thumbnails/' 21 thumbnails: 'test5/thumbnails/'
22 storyboards: 'test5/storyboards/'
22 torrents: 'test5/torrents/' 23 torrents: 'test5/torrents/'
23 captions: 'test5/captions/' 24 captions: 'test5/captions/'
24 cache: 'test5/cache/' 25 cache: 'test5/cache/'
diff --git a/config/test-6.yaml b/config/test-6.yaml
index a579f1f01..25efe0054 100644
--- a/config/test-6.yaml
+++ b/config/test-6.yaml
@@ -19,6 +19,7 @@ storage:
19 logs: 'test6/logs/' 19 logs: 'test6/logs/'
20 previews: 'test6/previews/' 20 previews: 'test6/previews/'
21 thumbnails: 'test6/thumbnails/' 21 thumbnails: 'test6/thumbnails/'
22 storyboards: 'test6/storyboards/'
22 torrents: 'test6/torrents/' 23 torrents: 'test6/torrents/'
23 captions: 'test6/captions/' 24 captions: 'test6/captions/'
24 cache: 'test6/cache/' 25 cache: 'test6/cache/'
diff --git a/config/test.yaml b/config/test.yaml
index 361064af1..41ec0917e 100644
--- a/config/test.yaml
+++ b/config/test.yaml
@@ -73,6 +73,8 @@ cache:
73 size: 1 73 size: 1
74 torrents: 74 torrents:
75 size: 1 75 size: 1
76 storyboards:
77 size: 1
76 78
77signup: 79signup:
78 enabled: true 80 enabled: true
diff --git a/server.ts b/server.ts
index a7a723b24..5d3acb2cd 100644
--- a/server.ts
+++ b/server.ts
@@ -101,7 +101,7 @@ loadLanguages()
101import { installApplication } from './server/initializers/installer' 101import { installApplication } from './server/initializers/installer'
102import { Emailer } from './server/lib/emailer' 102import { Emailer } from './server/lib/emailer'
103import { JobQueue } from './server/lib/job-queue' 103import { JobQueue } from './server/lib/job-queue'
104import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache' 104import { VideosPreviewCache, VideosCaptionCache, VideosStoryboardCache } from './server/lib/files-cache'
105import { 105import {
106 activityPubRouter, 106 activityPubRouter,
107 apiRouter, 107 apiRouter,
@@ -316,6 +316,7 @@ async function startApplication () {
316 VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE) 316 VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
317 VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE) 317 VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
318 VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE) 318 VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
319 VideosStoryboardCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE)
319 320
320 // Enable Schedulers 321 // Enable Schedulers
321 ActorFollowScheduler.Instance.enable() 322 ActorFollowScheduler.Instance.enable()
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 750e3091c..166fc2a22 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -33,7 +33,6 @@ import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '
33import { AccountModel } from '../../models/account/account' 33import { AccountModel } from '../../models/account/account'
34import { AccountVideoRateModel } from '../../models/account/account-video-rate' 34import { AccountVideoRateModel } from '../../models/account/account-video-rate'
35import { ActorFollowModel } from '../../models/actor/actor-follow' 35import { ActorFollowModel } from '../../models/actor/actor-follow'
36import { VideoCaptionModel } from '../../models/video/video-caption'
37import { VideoCommentModel } from '../../models/video/video-comment' 36import { VideoCommentModel } from '../../models/video/video-comment'
38import { VideoPlaylistModel } from '../../models/video/video-playlist' 37import { VideoPlaylistModel } from '../../models/video/video-playlist'
39import { VideoShareModel } from '../../models/video/video-share' 38import { VideoShareModel } from '../../models/video/video-share'
@@ -242,14 +241,13 @@ async function videoController (req: express.Request, res: express.Response) {
242 if (redirectIfNotOwned(video.url, res)) return 241 if (redirectIfNotOwned(video.url, res)) return
243 242
244 // We need captions to render AP object 243 // We need captions to render AP object
245 const captions = await VideoCaptionModel.listVideoCaptions(video.id) 244 const videoAP = await video.lightAPToFullAP(undefined)
246 const videoWithCaptions = Object.assign(video, { VideoCaptions: captions })
247 245
248 const audience = getAudience(videoWithCaptions.VideoChannel.Account.Actor, videoWithCaptions.privacy === VideoPrivacy.PUBLIC) 246 const audience = getAudience(videoAP.VideoChannel.Account.Actor, videoAP.privacy === VideoPrivacy.PUBLIC)
249 const videoObject = audiencify(await videoWithCaptions.toActivityPubObject(), audience) 247 const videoObject = audiencify(await videoAP.toActivityPubObject(), audience)
250 248
251 if (req.path.endsWith('/activity')) { 249 if (req.path.endsWith('/activity')) {
252 const data = buildCreateActivity(videoWithCaptions.url, video.VideoChannel.Account.Actor, videoObject, audience) 250 const data = buildCreateActivity(videoAP.url, video.VideoChannel.Account.Actor, videoObject, audience)
253 return activityPubResponse(activityPubContextify(data, 'Video'), res) 251 return activityPubResponse(activityPubContextify(data, 'Video'), res)
254 } 252 }
255 253
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 228eae109..c1f6756de 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -190,6 +190,9 @@ function customConfig (): CustomConfig {
190 }, 190 },
191 torrents: { 191 torrents: {
192 size: CONFIG.CACHE.TORRENTS.SIZE 192 size: CONFIG.CACHE.TORRENTS.SIZE
193 },
194 storyboards: {
195 size: CONFIG.CACHE.STORYBOARDS.SIZE
193 } 196 }
194 }, 197 },
195 signup: { 198 signup: {
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index d0eecf812..bbdda5b29 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -41,6 +41,7 @@ import { liveRouter } from './live'
41import { ownershipVideoRouter } from './ownership' 41import { ownershipVideoRouter } from './ownership'
42import { rateVideoRouter } from './rate' 42import { rateVideoRouter } from './rate'
43import { statsRouter } from './stats' 43import { statsRouter } from './stats'
44import { storyboardRouter } from './storyboard'
44import { studioRouter } from './studio' 45import { studioRouter } from './studio'
45import { tokenRouter } from './token' 46import { tokenRouter } from './token'
46import { transcodingRouter } from './transcoding' 47import { transcodingRouter } from './transcoding'
@@ -70,6 +71,7 @@ videosRouter.use('/', filesRouter)
70videosRouter.use('/', transcodingRouter) 71videosRouter.use('/', transcodingRouter)
71videosRouter.use('/', tokenRouter) 72videosRouter.use('/', tokenRouter)
72videosRouter.use('/', videoPasswordRouter) 73videosRouter.use('/', videoPasswordRouter)
74videosRouter.use('/', storyboardRouter)
73 75
74videosRouter.get('/categories', 76videosRouter.get('/categories',
75 openapiOperationDoc({ operationId: 'getCategories' }), 77 openapiOperationDoc({ operationId: 'getCategories' }),
diff --git a/server/controllers/api/videos/storyboard.ts b/server/controllers/api/videos/storyboard.ts
new file mode 100644
index 000000000..47a22011d
--- /dev/null
+++ b/server/controllers/api/videos/storyboard.ts
@@ -0,0 +1,29 @@
1import express from 'express'
2import { getVideoWithAttributes } from '@server/helpers/video'
3import { StoryboardModel } from '@server/models/video/storyboard'
4import { asyncMiddleware, videosGetValidator } from '../../../middlewares'
5
6const storyboardRouter = express.Router()
7
8storyboardRouter.get('/:id/storyboards',
9 asyncMiddleware(videosGetValidator),
10 asyncMiddleware(listStoryboards)
11)
12
13// ---------------------------------------------------------------------------
14
15export {
16 storyboardRouter
17}
18
19// ---------------------------------------------------------------------------
20
21async function listStoryboards (req: express.Request, res: express.Response) {
22 const video = getVideoWithAttributes(res)
23
24 const storyboards = await StoryboardModel.listStoryboardsOf(video)
25
26 return res.json({
27 storyboards: storyboards.map(s => s.toFormattedJSON())
28 })
29}
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 073eb480f..86ab4591e 100644
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -235,6 +235,15 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
235 }, 235 },
236 236
237 { 237 {
238 type: 'generate-video-storyboard' as 'generate-video-storyboard',
239 payload: {
240 videoUUID: video.uuid,
241 // No need to federate, we process these jobs sequentially
242 federate: false
243 }
244 },
245
246 {
238 type: 'notify', 247 type: 'notify',
239 payload: { 248 payload: {
240 action: 'new-video', 249 action: 'new-video',
diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts
index b082e41f6..6ffd39730 100644
--- a/server/controllers/lazy-static.ts
+++ b/server/controllers/lazy-static.ts
@@ -5,7 +5,7 @@ import { MActorImage } from '@server/types/models'
5import { HttpStatusCode } from '../../shared/models/http/http-error-codes' 5import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
6import { logger } from '../helpers/logger' 6import { logger } from '../helpers/logger'
7import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants' 7import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
8import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache' 8import { VideosCaptionCache, VideosPreviewCache, VideosStoryboardCache } from '../lib/files-cache'
9import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor' 9import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor'
10import { asyncMiddleware, handleStaticError } from '../middlewares' 10import { asyncMiddleware, handleStaticError } from '../middlewares'
11import { ActorImageModel } from '../models/actor/actor-image' 11import { ActorImageModel } from '../models/actor/actor-image'
@@ -33,6 +33,12 @@ lazyStaticRouter.use(
33) 33)
34 34
35lazyStaticRouter.use( 35lazyStaticRouter.use(
36 LAZY_STATIC_PATHS.STORYBOARDS + ':filename',
37 asyncMiddleware(getStoryboard),
38 handleStaticError
39)
40
41lazyStaticRouter.use(
36 LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename', 42 LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename',
37 asyncMiddleware(getVideoCaption), 43 asyncMiddleware(getVideoCaption),
38 handleStaticError 44 handleStaticError
@@ -126,6 +132,13 @@ async function getPreview (req: express.Request, res: express.Response) {
126 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) 132 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
127} 133}
128 134
135async function getStoryboard (req: express.Request, res: express.Response) {
136 const result = await VideosStoryboardCache.Instance.getFilePath(req.params.filename)
137 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
138
139 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
140}
141
129async function getVideoCaption (req: express.Request, res: express.Response) { 142async function getVideoCaption (req: express.Request, res: express.Response) {
130 const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename) 143 const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename)
131 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() 144 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 97b3577af..573a29754 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -1,6 +1,6 @@
1import validator from 'validator' 1import validator from 'validator'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@shared/models' 3import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models'
4import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' 4import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos'
5import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' 5import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
6import { peertubeTruncate } from '../../core-utils' 6import { peertubeTruncate } from '../../core-utils'
@@ -48,6 +48,10 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
48 logger.debug('Video has invalid icons', { video }) 48 logger.debug('Video has invalid icons', { video })
49 return false 49 return false
50 } 50 }
51 if (!setValidStoryboard(video)) {
52 logger.debug('Video has invalid preview (storyboard)', { video })
53 return false
54 }
51 55
52 // Default attributes 56 // Default attributes
53 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED 57 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
@@ -201,3 +205,36 @@ function setRemoteVideoContent (video: any) {
201 205
202 return true 206 return true
203} 207}
208
209function setValidStoryboard (video: VideoObject) {
210 if (!video.preview) return true
211 if (!Array.isArray(video.preview)) return false
212
213 video.preview = video.preview.filter(p => isStorybordValid(p))
214
215 return true
216}
217
218function isStorybordValid (preview: ActivityPubStoryboard) {
219 if (!preview) return false
220
221 if (
222 preview.type !== 'Image' ||
223 !isArray(preview.rel) ||
224 !preview.rel.includes('storyboard')
225 ) {
226 return false
227 }
228
229 preview.url = preview.url.filter(u => {
230 return u.mediaType === 'image/jpeg' &&
231 isActivityPubUrlValid(u.href) &&
232 validator.isInt(u.width + '', { min: 0 }) &&
233 validator.isInt(u.height + '', { min: 0 }) &&
234 validator.isInt(u.tileWidth + '', { min: 0 }) &&
235 validator.isInt(u.tileHeight + '', { min: 0 }) &&
236 isActivityPubVideoDurationValid(u.tileDuration)
237 })
238
239 return preview.url.length !== 0
240}
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 0a315ea70..939b73344 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -29,7 +29,8 @@ function checkMissedConfig () {
29 'video_channels.max_per_user', 29 'video_channels.max_per_user',
30 'csp.enabled', 'csp.report_only', 'csp.report_uri', 30 'csp.enabled', 'csp.report_only', 'csp.report_uri',
31 'security.frameguard.enabled', 'security.powered_by_header.enabled', 31 'security.frameguard.enabled', 'security.powered_by_header.enabled',
32 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'admin.email', 'contact_form.enabled', 32 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'cache.storyboards.size',
33 'admin.email', 'contact_form.enabled',
33 'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age', 34 'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age',
34 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 35 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
35 'redundancy.videos.strategies', 'redundancy.videos.check_interval', 36 'redundancy.videos.strategies', 'redundancy.videos.check_interval',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 51ac5d0ce..60ab6e204 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -112,6 +112,7 @@ const CONFIG = {
112 STREAMING_PLAYLISTS_DIR: buildPath(config.get<string>('storage.streaming_playlists')), 112 STREAMING_PLAYLISTS_DIR: buildPath(config.get<string>('storage.streaming_playlists')),
113 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), 113 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
114 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), 114 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
115 STORYBOARDS_DIR: buildPath(config.get<string>('storage.storyboards')),
115 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), 116 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
116 CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')), 117 CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),
117 TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')), 118 TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
@@ -482,6 +483,9 @@ const CONFIG = {
482 }, 483 },
483 TORRENTS: { 484 TORRENTS: {
484 get SIZE () { return config.get<number>('cache.torrents.size') } 485 get SIZE () { return config.get<number>('cache.torrents.size') }
486 },
487 STORYBOARDS: {
488 get SIZE () { return config.get<number>('cache.storyboards.size') }
485 } 489 }
486 }, 490 },
487 INSTANCE: { 491 INSTANCE: {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index e2f34fe16..3a643a60b 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -174,6 +174,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
174 'after-video-channel-import': 1, 174 'after-video-channel-import': 1,
175 'move-to-object-storage': 3, 175 'move-to-object-storage': 3,
176 'transcoding-job-builder': 1, 176 'transcoding-job-builder': 1,
177 'generate-video-storyboard': 1,
177 'notify': 1, 178 'notify': 1,
178 'federate-video': 1 179 'federate-video': 1
179} 180}
@@ -198,6 +199,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
198 'video-channel-import': 1, 199 'video-channel-import': 1,
199 'after-video-channel-import': 1, 200 'after-video-channel-import': 1,
200 'transcoding-job-builder': 1, 201 'transcoding-job-builder': 1,
202 'generate-video-storyboard': 1,
201 'notify': 5, 203 'notify': 5,
202 'federate-video': 3 204 'federate-video': 3
203} 205}
@@ -218,6 +220,7 @@ const JOB_TTL: { [id in JobType]: number } = {
218 'activitypub-refresher': 60000 * 10, // 10 minutes 220 'activitypub-refresher': 60000 * 10, // 10 minutes
219 'video-redundancy': 1000 * 3600 * 3, // 3 hours 221 'video-redundancy': 1000 * 3600 * 3, // 3 hours
220 'video-live-ending': 1000 * 60 * 10, // 10 minutes 222 'video-live-ending': 1000 * 60 * 10, // 10 minutes
223 'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes
221 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours 224 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
222 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours 225 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
223 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours 226 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
@@ -766,7 +769,8 @@ const LAZY_STATIC_PATHS = {
766 AVATARS: '/lazy-static/avatars/', 769 AVATARS: '/lazy-static/avatars/',
767 PREVIEWS: '/lazy-static/previews/', 770 PREVIEWS: '/lazy-static/previews/',
768 VIDEO_CAPTIONS: '/lazy-static/video-captions/', 771 VIDEO_CAPTIONS: '/lazy-static/video-captions/',
769 TORRENTS: '/lazy-static/torrents/' 772 TORRENTS: '/lazy-static/torrents/',
773 STORYBOARDS: '/lazy-static/storyboards/'
770} 774}
771const OBJECT_STORAGE_PROXY_PATHS = { 775const OBJECT_STORAGE_PROXY_PATHS = {
772 PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/', 776 PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/',
@@ -813,6 +817,14 @@ const ACTOR_IMAGES_SIZE: { [key in ActorImageType]: { width: number, height: num
813 ] 817 ]
814} 818}
815 819
820const STORYBOARD = {
821 SPRITE_SIZE: {
822 width: 192,
823 height: 108
824 },
825 SPRITES_MAX_EDGE_COUNT: 10
826}
827
816const EMBED_SIZE = { 828const EMBED_SIZE = {
817 width: 560, 829 width: 560,
818 height: 315 830 height: 315
@@ -824,6 +836,10 @@ const FILES_CACHE = {
824 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'), 836 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
825 MAX_AGE: 1000 * 3600 * 3 // 3 hours 837 MAX_AGE: 1000 * 3600 * 3 // 3 hours
826 }, 838 },
839 STORYBOARDS: {
840 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'storyboards'),
841 MAX_AGE: 1000 * 3600 * 24 // 24 hours
842 },
827 VIDEO_CAPTIONS: { 843 VIDEO_CAPTIONS: {
828 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'), 844 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'),
829 MAX_AGE: 1000 * 3600 * 3 // 3 hours 845 MAX_AGE: 1000 * 3600 * 3 // 3 hours
@@ -1090,6 +1106,7 @@ export {
1090 RESUMABLE_UPLOAD_SESSION_LIFETIME, 1106 RESUMABLE_UPLOAD_SESSION_LIFETIME,
1091 RUNNER_JOB_STATES, 1107 RUNNER_JOB_STATES,
1092 P2P_MEDIA_LOADER_PEER_VERSION, 1108 P2P_MEDIA_LOADER_PEER_VERSION,
1109 STORYBOARD,
1093 ACTOR_IMAGES_SIZE, 1110 ACTOR_IMAGES_SIZE,
1094 ACCEPT_HEADERS, 1111 ACCEPT_HEADERS,
1095 BCRYPT_SALT_SIZE, 1112 BCRYPT_SALT_SIZE,
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 9e926c26c..bc120e398 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -10,6 +10,7 @@ import { UserModel } from '@server/models/user/user'
10import { UserNotificationModel } from '@server/models/user/user-notification' 10import { UserNotificationModel } from '@server/models/user/user-notification'
11import { UserRegistrationModel } from '@server/models/user/user-registration' 11import { UserRegistrationModel } from '@server/models/user/user-registration'
12import { UserVideoHistoryModel } from '@server/models/user/user-video-history' 12import { UserVideoHistoryModel } from '@server/models/user/user-video-history'
13import { StoryboardModel } from '@server/models/video/storyboard'
13import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 14import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
14import { VideoJobInfoModel } from '@server/models/video/video-job-info' 15import { VideoJobInfoModel } from '@server/models/video/video-job-info'
15import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' 16import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
@@ -167,7 +168,8 @@ async function initDatabaseModels (silent: boolean) {
167 VideoPasswordModel, 168 VideoPasswordModel,
168 RunnerRegistrationTokenModel, 169 RunnerRegistrationTokenModel,
169 RunnerModel, 170 RunnerModel,
170 RunnerJobModel 171 RunnerJobModel,
172 StoryboardModel
171 ]) 173 ])
172 174
173 // Check extensions exist in the database 175 // Check extensions exist in the database
diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts
index a3ca52a31..750276a11 100644
--- a/server/lib/activitypub/context.ts
+++ b/server/lib/activitypub/context.ts
@@ -46,6 +46,19 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string
46 46
47 Infohash: 'pt:Infohash', 47 Infohash: 'pt:Infohash',
48 48
49 tileWidth: {
50 '@type': 'sc:Number',
51 '@id': 'pt:tileWidth'
52 },
53 tileHeight: {
54 '@type': 'sc:Number',
55 '@id': 'pt:tileHeight'
56 },
57 tileDuration: {
58 '@type': 'sc:Number',
59 '@id': 'pt:tileDuration'
60 },
61
49 originallyPublishedAt: 'sc:datePublished', 62 originallyPublishedAt: 'sc:datePublished',
50 views: { 63 views: {
51 '@type': 'sc:Number', 64 '@type': 'sc:Number',
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index 379e2d9d8..3d2b437e4 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -10,8 +10,7 @@ import {
10 MActor, 10 MActor,
11 MActorLight, 11 MActorLight,
12 MChannelDefault, 12 MChannelDefault,
13 MVideoAP, 13 MVideoAPLight,
14 MVideoAPWithoutCaption,
15 MVideoPlaylistFull, 14 MVideoPlaylistFull,
16 MVideoRedundancyVideo 15 MVideoRedundancyVideo
17} from '../../../types/models' 16} from '../../../types/models'
@@ -20,10 +19,10 @@ import { getUpdateActivityPubUrl } from '../url'
20import { getActorsInvolvedInVideo } from './shared' 19import { getActorsInvolvedInVideo } from './shared'
21import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils' 20import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils'
22 21
23async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: Transaction, overriddenByActor?: MActor) { 22async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) {
24 const video = videoArg as MVideoAP 23 if (!videoArg.hasPrivacyForFederation()) return undefined
25 24
26 if (!video.hasPrivacyForFederation()) return undefined 25 const video = await videoArg.lightAPToFullAP(transaction)
27 26
28 logger.info('Creating job to update video %s.', video.url) 27 logger.info('Creating job to update video %s.', video.url)
29 28
@@ -31,11 +30,6 @@ async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: T
31 30
32 const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) 31 const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString())
33 32
34 // Needed to build the AP object
35 if (!video.VideoCaptions) {
36 video.VideoCaptions = await video.$get('VideoCaptions', { transaction })
37 }
38
39 const videoObject = await video.toActivityPubObject() 33 const videoObject = await video.toActivityPubObject()
40 const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) 34 const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
41 35
diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts
index bd0c54b0c..d7e251153 100644
--- a/server/lib/activitypub/videos/federate.ts
+++ b/server/lib/activitypub/videos/federate.ts
@@ -1,10 +1,9 @@
1import { Transaction } from 'sequelize/types' 1import { Transaction } from 'sequelize/types'
2import { isArray } from '@server/helpers/custom-validators/misc' 2import { MVideoAP, MVideoAPLight } from '@server/types/models'
3import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models'
4import { sendCreateVideo, sendUpdateVideo } from '../send' 3import { sendCreateVideo, sendUpdateVideo } from '../send'
5import { shareVideoByServerAndChannel } from '../share' 4import { shareVideoByServerAndChannel } from '../share'
6 5
7async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) { 6async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) {
8 const video = videoArg as MVideoAP 7 const video = videoArg as MVideoAP
9 8
10 if ( 9 if (
@@ -13,13 +12,7 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid
13 // Check the video is public/unlisted and published 12 // Check the video is public/unlisted and published
14 video.hasPrivacyForFederation() && video.hasStateForFederation() 13 video.hasPrivacyForFederation() && video.hasStateForFederation()
15 ) { 14 ) {
16 // Fetch more attributes that we will need to serialize in AP object 15 const video = await videoArg.lightAPToFullAP(transaction)
17 if (isArray(video.VideoCaptions) === false) {
18 video.VideoCaptions = await video.$get('VideoCaptions', {
19 attributes: [ 'filename', 'language' ],
20 transaction
21 })
22 }
23 16
24 if (isNewVideo) { 17 if (isNewVideo) {
25 // Now we'll add the video's meta data to our followers 18 // Now we'll add the video's meta data to our followers
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
index c0b92c93d..7c5c73139 100644
--- a/server/lib/activitypub/videos/shared/abstract-builder.ts
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -3,6 +3,7 @@ import { deleteAllModels, filterNonExistingModels } from '@server/helpers/databa
3import { logger, LoggerTagsFn } from '@server/helpers/logger' 3import { logger, LoggerTagsFn } from '@server/helpers/logger'
4import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' 4import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
5import { setVideoTags } from '@server/lib/video' 5import { setVideoTags } from '@server/lib/video'
6import { StoryboardModel } from '@server/models/video/storyboard'
6import { VideoCaptionModel } from '@server/models/video/video-caption' 7import { VideoCaptionModel } from '@server/models/video/video-caption'
7import { VideoFileModel } from '@server/models/video/video-file' 8import { VideoFileModel } from '@server/models/video/video-file'
8import { VideoLiveModel } from '@server/models/video/video-live' 9import { VideoLiveModel } from '@server/models/video/video-live'
@@ -24,6 +25,7 @@ import {
24 getFileAttributesFromUrl, 25 getFileAttributesFromUrl,
25 getLiveAttributesFromObject, 26 getLiveAttributesFromObject,
26 getPreviewFromIcons, 27 getPreviewFromIcons,
28 getStoryboardAttributeFromObject,
27 getStreamingPlaylistAttributesFromObject, 29 getStreamingPlaylistAttributesFromObject,
28 getTagsFromObject, 30 getTagsFromObject,
29 getThumbnailFromIcons 31 getThumbnailFromIcons
@@ -107,6 +109,16 @@ export abstract class APVideoAbstractBuilder {
107 } 109 }
108 } 110 }
109 111
112 protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) {
113 const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t)
114 if (existingStoryboard) await existingStoryboard.destroy({ transaction: t })
115
116 const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject)
117 if (!storyboardAttributes) return
118
119 return StoryboardModel.create(storyboardAttributes, { transaction: t })
120 }
121
110 protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { 122 protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) {
111 const attributes = getLiveAttributesFromObject(video, this.videoObject) 123 const attributes = getLiveAttributesFromObject(video, this.videoObject)
112 const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) 124 const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true })
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts
index 77321d8a5..e6d7bc23c 100644
--- a/server/lib/activitypub/videos/shared/creator.ts
+++ b/server/lib/activitypub/videos/shared/creator.ts
@@ -48,6 +48,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
48 await this.setTrackers(videoCreated, t) 48 await this.setTrackers(videoCreated, t)
49 await this.insertOrReplaceCaptions(videoCreated, t) 49 await this.insertOrReplaceCaptions(videoCreated, t)
50 await this.insertOrReplaceLive(videoCreated, t) 50 await this.insertOrReplaceLive(videoCreated, t)
51 await this.insertOrReplaceStoryboard(videoCreated, t)
51 52
52 // We added a video in this channel, set it as updated 53 // We added a video in this channel, set it as updated
53 await channel.setAsUpdated(t) 54 await channel.setAsUpdated(t)
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index 8fd0a816c..a9e0bed97 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -1,6 +1,6 @@
1import { maxBy, minBy } from 'lodash' 1import { maxBy, minBy } from 'lodash'
2import { decode as magnetUriDecode } from 'magnet-uri' 2import { decode as magnetUriDecode } from 'magnet-uri'
3import { basename } from 'path' 3import { basename, extname } from 'path'
4import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos' 4import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos'
5import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos' 5import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos'
6import { logger } from '@server/helpers/logger' 6import { logger } from '@server/helpers/logger'
@@ -25,6 +25,9 @@ import {
25 VideoStreamingPlaylistType 25 VideoStreamingPlaylistType
26} from '@shared/models' 26} from '@shared/models'
27import { getDurationFromActivityStream } from '../../activity' 27import { getDurationFromActivityStream } from '../../activity'
28import { isArray } from '@server/helpers/custom-validators/misc'
29import { generateImageFilename } from '@server/helpers/image-utils'
30import { arrayify } from '@shared/core-utils'
28 31
29function getThumbnailFromIcons (videoObject: VideoObject) { 32function getThumbnailFromIcons (videoObject: VideoObject) {
30 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) 33 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@@ -166,6 +169,26 @@ function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObje
166 })) 169 }))
167} 170}
168 171
172function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) {
173 if (!isArray(videoObject.preview)) return undefined
174
175 const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard'))
176 if (!storyboard) return undefined
177
178 const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg')
179
180 return {
181 filename: generateImageFilename(extname(url.href)),
182 totalHeight: url.height,
183 totalWidth: url.width,
184 spriteHeight: url.tileHeight,
185 spriteWidth: url.tileWidth,
186 spriteDuration: getDurationFromActivityStream(url.tileDuration),
187 fileUrl: url.href,
188 videoId: video.id
189 }
190}
191
169function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { 192function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
170 const privacy = to.includes(ACTIVITY_PUB.PUBLIC) 193 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
171 ? VideoPrivacy.PUBLIC 194 ? VideoPrivacy.PUBLIC
@@ -228,6 +251,7 @@ export {
228 251
229 getLiveAttributesFromObject, 252 getLiveAttributesFromObject,
230 getCaptionAttributesFromObject, 253 getCaptionAttributesFromObject,
254 getStoryboardAttributeFromObject,
231 255
232 getVideoAttributesFromObject 256 getVideoAttributesFromObject
233} 257}
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts
index 6ddd2301b..3a0886523 100644
--- a/server/lib/activitypub/videos/updater.ts
+++ b/server/lib/activitypub/videos/updater.ts
@@ -57,6 +57,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
57 await Promise.all([ 57 await Promise.all([
58 runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), 58 runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
59 runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), 59 runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
60 runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)),
60 this.setOrDeleteLive(videoUpdated), 61 this.setOrDeleteLive(videoUpdated),
61 this.setPreview(videoUpdated) 62 this.setPreview(videoUpdated)
62 ]) 63 ])
@@ -138,6 +139,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
138 await this.insertOrReplaceCaptions(videoUpdated, t) 139 await this.insertOrReplaceCaptions(videoUpdated, t)
139 } 140 }
140 141
142 private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) {
143 await this.insertOrReplaceStoryboard(videoUpdated, t)
144 }
145
141 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { 146 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
142 if (!this.video.isLive) return 147 if (!this.video.isLive) return
143 148
diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts
index e5853f7d6..59cec7215 100644
--- a/server/lib/files-cache/index.ts
+++ b/server/lib/files-cache/index.ts
@@ -1,3 +1,4 @@
1export * from './videos-preview-cache'
2export * from './videos-caption-cache' 1export * from './videos-caption-cache'
2export * from './videos-preview-cache'
3export * from './videos-storyboard-cache'
3export * from './videos-torrent-cache' 4export * from './videos-torrent-cache'
diff --git a/server/lib/files-cache/videos-storyboard-cache.ts b/server/lib/files-cache/videos-storyboard-cache.ts
new file mode 100644
index 000000000..b0a55104f
--- /dev/null
+++ b/server/lib/files-cache/videos-storyboard-cache.ts
@@ -0,0 +1,53 @@
1import { join } from 'path'
2import { logger } from '@server/helpers/logger'
3import { doRequestAndSaveToFile } from '@server/helpers/requests'
4import { StoryboardModel } from '@server/models/video/storyboard'
5import { FILES_CACHE } from '../../initializers/constants'
6import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
7
8class VideosStoryboardCache extends AbstractVideoStaticFileCache <string> {
9
10 private static instance: VideosStoryboardCache
11
12 private constructor () {
13 super()
14 }
15
16 static get Instance () {
17 return this.instance || (this.instance = new this())
18 }
19
20 async getFilePathImpl (filename: string) {
21 const storyboard = await StoryboardModel.loadWithVideoByFilename(filename)
22 if (!storyboard) return undefined
23
24 if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() }
25
26 return this.loadRemoteFile(storyboard.filename)
27 }
28
29 // Key is the storyboard filename
30 protected async loadRemoteFile (key: string) {
31 const storyboard = await StoryboardModel.loadWithVideoByFilename(key)
32 if (!storyboard) return undefined
33
34 const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename)
35 const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video)
36
37 try {
38 await doRequestAndSaveToFile(remoteUrl, destPath)
39
40 logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath)
41
42 return { isOwned: false, path: destPath }
43 } catch (err) {
44 logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err })
45
46 return undefined
47 }
48 }
49}
50
51export {
52 VideosStoryboardCache
53}
diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts
new file mode 100644
index 000000000..652cac272
--- /dev/null
+++ b/server/lib/job-queue/handlers/generate-storyboard.ts
@@ -0,0 +1,138 @@
1import { Job } from 'bullmq'
2import { join } from 'path'
3import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
4import { generateImageFilename, getImageSize } from '@server/helpers/image-utils'
5import { logger, loggerTagsFactory } from '@server/helpers/logger'
6import { CONFIG } from '@server/initializers/config'
7import { STORYBOARD } from '@server/initializers/constants'
8import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
9import { VideoPathManager } from '@server/lib/video-path-manager'
10import { StoryboardModel } from '@server/models/video/storyboard'
11import { VideoModel } from '@server/models/video/video'
12import { MVideo } from '@server/types/models'
13import { FFmpegImage, isAudioFile } from '@shared/ffmpeg'
14import { GenerateStoryboardPayload } from '@shared/models'
15
16const lTagsBase = loggerTagsFactory('storyboard')
17
18async function processGenerateStoryboard (job: Job): Promise<void> {
19 const payload = job.data as GenerateStoryboardPayload
20 const lTags = lTagsBase(payload.videoUUID)
21
22 logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags)
23
24 const video = await VideoModel.loadFull(payload.videoUUID)
25 if (!video) {
26 logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags)
27 return
28 }
29
30 const inputFile = video.getMaxQualityFile()
31
32 await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
33 const isAudio = await isAudioFile(videoPath)
34
35 if (isAudio) {
36 logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
37 return
38 }
39
40 const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
41
42 const filename = generateImageFilename()
43 const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
44
45 const totalSprites = buildTotalSprites(video)
46 const spriteDuration = Math.round(video.duration / totalSprites)
47
48 const spritesCount = findGridSize({
49 toFind: totalSprites,
50 maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
51 })
52
53 logger.debug(
54 'Generating storyboard from video of %s to %s', video.uuid, destination,
55 { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration }
56 )
57
58 await ffmpeg.generateStoryboardFromVideo({
59 destination,
60 path: videoPath,
61 sprites: {
62 size: STORYBOARD.SPRITE_SIZE,
63 count: spritesCount,
64 duration: spriteDuration
65 }
66 })
67
68 const imageSize = await getImageSize(destination)
69
70 const existing = await StoryboardModel.loadByVideo(video.id)
71 if (existing) await existing.destroy()
72
73 await StoryboardModel.create({
74 filename,
75 totalHeight: imageSize.height,
76 totalWidth: imageSize.width,
77 spriteHeight: STORYBOARD.SPRITE_SIZE.height,
78 spriteWidth: STORYBOARD.SPRITE_SIZE.width,
79 spriteDuration,
80 videoId: video.id
81 })
82
83 logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags)
84 })
85
86 if (payload.federate) {
87 await federateVideoIfNeeded(video, false)
88 }
89}
90
91// ---------------------------------------------------------------------------
92
93export {
94 processGenerateStoryboard
95}
96
97function buildTotalSprites (video: MVideo) {
98 const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width
99 const totalSprites = Math.min(Math.ceil(video.duration), maxSprites)
100
101 // We can generate a single line
102 if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites
103
104 return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT)
105}
106
107function findGridSize (options: {
108 toFind: number
109 maxEdgeCount: number
110}) {
111 const { toFind, maxEdgeCount } = options
112
113 for (let i = 1; i <= maxEdgeCount; i++) {
114 for (let j = i; j <= maxEdgeCount; j++) {
115 if (toFind === i * j) return { width: j, height: i }
116 }
117 }
118
119 throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
120}
121
122function findGridFit (value: number, maxMultiplier: number) {
123 for (let i = value; i--; i > 0) {
124 if (!isPrimeWithin(i, maxMultiplier)) return i
125 }
126
127 throw new Error('Could not find prime number below ' + value)
128}
129
130function isPrimeWithin (value: number, maxMultiplier: number) {
131 if (value < 2) return false
132
133 for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) {
134 if (value % i === 0 && value / i <= maxMultiplier) return false
135 }
136
137 return true
138}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index cdd362f6e..c1355dcef 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -306,6 +306,15 @@ async function afterImportSuccess (options: {
306 Notifier.Instance.notifyOnNewVideoIfNeeded(video) 306 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
307 } 307 }
308 308
309 // Generate the storyboard in the job queue, and don't forget to federate an update after
310 await JobQueue.Instance.createJob({
311 type: 'generate-video-storyboard' as 'generate-video-storyboard',
312 payload: {
313 videoUUID: video.uuid,
314 federate: true
315 }
316 })
317
309 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { 318 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
310 await JobQueue.Instance.createJob( 319 await JobQueue.Instance.createJob(
311 await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) 320 await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 49feb53f2..95d4f5e64 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -1,6 +1,8 @@
1import { Job } from 'bullmq' 1import { Job } from 'bullmq'
2import { readdir, remove } from 'fs-extra' 2import { readdir, remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { peertubeTruncate } from '@server/helpers/core-utils'
5import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
4import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 6import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
5import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' 7import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
6import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' 8import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
@@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv
20import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' 22import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'
21import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 23import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
22import { logger, loggerTagsFactory } from '../../../helpers/logger' 24import { logger, loggerTagsFactory } from '../../../helpers/logger'
23import { peertubeTruncate } from '@server/helpers/core-utils' 25import { JobQueue } from '../job-queue'
24import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
25 26
26const lTags = loggerTagsFactory('live', 'job') 27const lTags = loggerTagsFactory('live', 'job')
27 28
@@ -147,6 +148,8 @@ async function saveReplayToExternalVideo (options: {
147 } 148 }
148 149
149 await moveToNextState({ video: replayVideo, isNewVideo: true }) 150 await moveToNextState({ video: replayVideo, isNewVideo: true })
151
152 await createStoryboardJob(replayVideo)
150} 153}
151 154
152async function replaceLiveByReplay (options: { 155async function replaceLiveByReplay (options: {
@@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: {
186 189
187 await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) 190 await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
188 191
192 // FIXME: should not happen in this function
189 if (permanentLive) { // Remove session replay 193 if (permanentLive) { // Remove session replay
190 await remove(replayDirectory) 194 await remove(replayDirectory)
191 } else { // We won't stream again in this live, we can delete the base replay directory 195 } else { // We won't stream again in this live, we can delete the base replay directory
@@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: {
213 217
214 // We consider this is a new video 218 // We consider this is a new video
215 await moveToNextState({ video: videoWithFiles, isNewVideo: true }) 219 await moveToNextState({ video: videoWithFiles, isNewVideo: true })
220
221 await createStoryboardJob(videoWithFiles)
216} 222}
217 223
218async function assignReplayFilesToVideo (options: { 224async function assignReplayFilesToVideo (options: {
@@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: {
277 logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) 283 logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
278 } 284 }
279} 285}
286
287function createStoryboardJob (video: MVideo) {
288 return JobQueue.Instance.createJob({
289 type: 'generate-video-storyboard' as 'generate-video-storyboard',
290 payload: {
291 videoUUID: video.uuid,
292 federate: true
293 }
294 })
295}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 03f6fbea7..177bca285 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -25,6 +25,7 @@ import {
25 DeleteResumableUploadMetaFilePayload, 25 DeleteResumableUploadMetaFilePayload,
26 EmailPayload, 26 EmailPayload,
27 FederateVideoPayload, 27 FederateVideoPayload,
28 GenerateStoryboardPayload,
28 JobState, 29 JobState,
29 JobType, 30 JobType,
30 ManageVideoTorrentPayload, 31 ManageVideoTorrentPayload,
@@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending'
65import { processVideoStudioEdition } from './handlers/video-studio-edition' 66import { processVideoStudioEdition } from './handlers/video-studio-edition'
66import { processVideoTranscoding } from './handlers/video-transcoding' 67import { processVideoTranscoding } from './handlers/video-transcoding'
67import { processVideosViewsStats } from './handlers/video-views-stats' 68import { processVideosViewsStats } from './handlers/video-views-stats'
69import { processGenerateStoryboard } from './handlers/generate-storyboard'
68 70
69export type CreateJobArgument = 71export type CreateJobArgument =
70 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 72 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -91,7 +93,8 @@ export type CreateJobArgument =
91 { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | 93 { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
92 { type: 'notify', payload: NotifyPayload } | 94 { type: 'notify', payload: NotifyPayload } |
93 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | 95 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
94 { type: 'federate-video', payload: FederateVideoPayload } 96 { type: 'federate-video', payload: FederateVideoPayload } |
97 { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
95 98
96export type CreateJobOptions = { 99export type CreateJobOptions = {
97 delay?: number 100 delay?: number
@@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
122 'video-redundancy': processVideoRedundancy, 125 'video-redundancy': processVideoRedundancy,
123 'video-studio-edition': processVideoStudioEdition, 126 'video-studio-edition': processVideoStudioEdition,
124 'video-transcoding': processVideoTranscoding, 127 'video-transcoding': processVideoTranscoding,
125 'videos-views-stats': processVideosViewsStats 128 'videos-views-stats': processVideosViewsStats,
129 'generate-video-storyboard': processGenerateStoryboard
126} 130}
127 131
128const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { 132const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
@@ -141,10 +145,11 @@ const jobTypes: JobType[] = [
141 'after-video-channel-import', 145 'after-video-channel-import',
142 'email', 146 'email',
143 'federate-video', 147 'federate-video',
144 'transcoding-job-builder', 148 'generate-video-storyboard',
145 'manage-video-torrent', 149 'manage-video-torrent',
146 'move-to-object-storage', 150 'move-to-object-storage',
147 'notify', 151 'notify',
152 'transcoding-job-builder',
148 'video-channel-import', 153 'video-channel-import',
149 'video-file-import', 154 'video-file-import',
150 'video-import', 155 'video-import',
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 8430b2227..48d9986b5 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -325,8 +325,8 @@ class Redis {
325 const value = await this.getValue('resumable-upload-' + uploadId) 325 const value = await this.getValue('resumable-upload-' + uploadId)
326 326
327 return value 327 return value
328 ? JSON.parse(value) 328 ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } }
329 : '' 329 : undefined
330 } 330 }
331 331
332 deleteUploadSession (uploadId: string) { 332 deleteUploadSession (uploadId: string) {
diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts
index 7cc8f20bc..a499db422 100644
--- a/server/lib/transcoding/web-transcoding.ts
+++ b/server/lib/transcoding/web-transcoding.ts
@@ -9,6 +9,7 @@ import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVOD
9import { VideoResolution, VideoStorage } from '@shared/models' 9import { VideoResolution, VideoStorage } from '@shared/models'
10import { CONFIG } from '../../initializers/config' 10import { CONFIG } from '../../initializers/config'
11import { VideoFileModel } from '../../models/video/video-file' 11import { VideoFileModel } from '../../models/video/video-file'
12import { JobQueue } from '../job-queue'
12import { generateWebTorrentVideoFilename } from '../paths' 13import { generateWebTorrentVideoFilename } from '../paths'
13import { buildFileMetadata } from '../video-file' 14import { buildFileMetadata } from '../video-file'
14import { VideoPathManager } from '../video-path-manager' 15import { VideoPathManager } from '../video-path-manager'
@@ -198,7 +199,8 @@ export async function mergeAudioVideofile (options: {
198 return onWebTorrentVideoFileTranscoding({ 199 return onWebTorrentVideoFileTranscoding({
199 video, 200 video,
200 videoFile: inputVideoFile, 201 videoFile: inputVideoFile,
201 videoOutputPath 202 videoOutputPath,
203 wasAudioFile: true
202 }) 204 })
203 }) 205 })
204 206
@@ -212,8 +214,9 @@ export async function onWebTorrentVideoFileTranscoding (options: {
212 video: MVideoFullLight 214 video: MVideoFullLight
213 videoFile: MVideoFile 215 videoFile: MVideoFile
214 videoOutputPath: string 216 videoOutputPath: string
217 wasAudioFile?: boolean // default false
215}) { 218}) {
216 const { video, videoFile, videoOutputPath } = options 219 const { video, videoFile, videoOutputPath, wasAudioFile } = options
217 220
218 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) 221 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
219 222
@@ -242,6 +245,17 @@ export async function onWebTorrentVideoFileTranscoding (options: {
242 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 245 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
243 video.VideoFiles = await video.$get('VideoFiles') 246 video.VideoFiles = await video.$get('VideoFiles')
244 247
248 if (wasAudioFile) {
249 await JobQueue.Instance.createJob({
250 type: 'generate-video-storyboard' as 'generate-video-storyboard',
251 payload: {
252 videoUUID: video.uuid,
253 // No need to federate, we process these jobs sequentially
254 federate: false
255 }
256 })
257 }
258
245 return { video, videoFile } 259 return { video, videoFile }
246 } finally { 260 } finally {
247 mutexReleaser() 261 mutexReleaser()
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index a0074cb24..7029a857f 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -25,6 +25,7 @@ const customConfigUpdateValidator = [
25 body('cache.previews.size').isInt(), 25 body('cache.previews.size').isInt(),
26 body('cache.captions.size').isInt(), 26 body('cache.captions.size').isInt(),
27 body('cache.torrents.size').isInt(), 27 body('cache.torrents.size').isInt(),
28 body('cache.storyboards.size').isInt(),
28 29
29 body('signup.enabled').isBoolean(), 30 body('signup.enabled').isBoolean(),
30 body('signup.limit').isInt(), 31 body('signup.limit').isInt(),
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index f2001e432..4179545b8 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -5,6 +5,7 @@ import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
5import { VideoViewsManager } from '@server/lib/views/video-views-manager' 5import { VideoViewsManager } from '@server/lib/views/video-views-manager'
6import { uuidToShort } from '@shared/extra-utils' 6import { uuidToShort } from '@shared/extra-utils'
7import { 7import {
8 ActivityPubStoryboard,
8 ActivityTagObject, 9 ActivityTagObject,
9 ActivityUrlObject, 10 ActivityUrlObject,
10 Video, 11 Video,
@@ -347,29 +348,17 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
347 name: t.name 348 name: t.name
348 })) 349 }))
349 350
350 let language 351 const language = video.language
351 if (video.language) { 352 ? { identifier: video.language, name: getLanguageLabel(video.language) }
352 language = { 353 : undefined
353 identifier: video.language,
354 name: getLanguageLabel(video.language)
355 }
356 }
357 354
358 let category 355 const category = video.category
359 if (video.category) { 356 ? { identifier: video.category + '', name: getCategoryLabel(video.category) }
360 category = { 357 : undefined
361 identifier: video.category + '',
362 name: getCategoryLabel(video.category)
363 }
364 }
365 358
366 let licence 359 const licence = video.licence
367 if (video.licence) { 360 ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) }
368 licence = { 361 : undefined
369 identifier: video.licence + '',
370 name: getLicenceLabel(video.licence)
371 }
372 }
373 362
374 const url: ActivityUrlObject[] = [ 363 const url: ActivityUrlObject[] = [
375 // HTML url should be the first element in the array so Mastodon correctly displays the embed 364 // HTML url should be the first element in the array so Mastodon correctly displays the embed
@@ -465,6 +454,8 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
465 height: i.height 454 height: i.height
466 })), 455 })),
467 456
457 preview: buildPreviewAPAttribute(video),
458
468 url, 459 url,
469 460
470 likes: getLocalVideoLikesActivityPubUrl(video), 461 likes: getLocalVideoLikesActivityPubUrl(video),
@@ -541,3 +532,30 @@ function buildLiveAPAttributes (video: MVideoAP) {
541 latencyMode: video.VideoLive.latencyMode 532 latencyMode: video.VideoLive.latencyMode
542 } 533 }
543} 534}
535
536function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {
537 if (!video.Storyboard) return undefined
538
539 const storyboard = video.Storyboard
540
541 return [
542 {
543 type: 'Image',
544 rel: [ 'storyboard' ],
545 url: [
546 {
547 mediaType: 'image/jpeg',
548
549 href: storyboard.getOriginFileUrl(video),
550
551 width: storyboard.totalWidth,
552 height: storyboard.totalHeight,
553
554 tileWidth: storyboard.spriteWidth,
555 tileHeight: storyboard.spriteHeight,
556 tileDuration: getActivityStreamDuration(storyboard.spriteDuration)
557 }
558 ]
559 }
560 ]
561}
diff --git a/server/models/video/storyboard.ts b/server/models/video/storyboard.ts
new file mode 100644
index 000000000..65a044c98
--- /dev/null
+++ b/server/models/video/storyboard.ts
@@ -0,0 +1,169 @@
1import { remove } from 'fs-extra'
2import { join } from 'path'
3import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
4import { CONFIG } from '@server/initializers/config'
5import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models'
6import { Storyboard } from '@shared/models'
7import { AttributesOnly } from '@shared/typescript-utils'
8import { logger } from '../../helpers/logger'
9import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
10import { VideoModel } from './video'
11import { Transaction } from 'sequelize'
12
13@Table({
14 tableName: 'storyboard',
15 indexes: [
16 {
17 fields: [ 'videoId' ],
18 unique: true
19 },
20 {
21 fields: [ 'filename' ],
22 unique: true
23 }
24 ]
25})
26export class StoryboardModel extends Model<Partial<AttributesOnly<StoryboardModel>>> {
27
28 @AllowNull(false)
29 @Column
30 filename: string
31
32 @AllowNull(false)
33 @Column
34 totalHeight: number
35
36 @AllowNull(false)
37 @Column
38 totalWidth: number
39
40 @AllowNull(false)
41 @Column
42 spriteHeight: number
43
44 @AllowNull(false)
45 @Column
46 spriteWidth: number
47
48 @AllowNull(false)
49 @Column
50 spriteDuration: number
51
52 @AllowNull(true)
53 @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max))
54 fileUrl: string
55
56 @ForeignKey(() => VideoModel)
57 @Column
58 videoId: number
59
60 @BelongsTo(() => VideoModel, {
61 foreignKey: {
62 allowNull: true
63 },
64 onDelete: 'CASCADE'
65 })
66 Video: VideoModel
67
68 @CreatedAt
69 createdAt: Date
70
71 @UpdatedAt
72 updatedAt: Date
73
74 @AfterDestroy
75 static removeInstanceFile (instance: StoryboardModel) {
76 logger.info('Removing storyboard file %s.', instance.filename)
77
78 // Don't block the transaction
79 instance.removeFile()
80 .catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err }))
81 }
82
83 static loadByVideo (videoId: number, transaction?: Transaction): Promise<MStoryboard> {
84 const query = {
85 where: {
86 videoId
87 },
88 transaction
89 }
90
91 return StoryboardModel.findOne(query)
92 }
93
94 static loadByFilename (filename: string): Promise<MStoryboard> {
95 const query = {
96 where: {
97 filename
98 }
99 }
100
101 return StoryboardModel.findOne(query)
102 }
103
104 static loadWithVideoByFilename (filename: string): Promise<MStoryboardVideo> {
105 const query = {
106 where: {
107 filename
108 },
109 include: [
110 {
111 model: VideoModel.unscoped(),
112 required: true
113 }
114 ]
115 }
116
117 return StoryboardModel.findOne(query)
118 }
119
120 // ---------------------------------------------------------------------------
121
122 static async listStoryboardsOf (video: MVideo): Promise<MStoryboardVideo[]> {
123 const query = {
124 where: {
125 videoId: video.id
126 }
127 }
128
129 const storyboards = await StoryboardModel.findAll<MStoryboard>(query)
130
131 return storyboards.map(s => Object.assign(s, { Video: video }))
132 }
133
134 // ---------------------------------------------------------------------------
135
136 getOriginFileUrl (video: MVideo) {
137 if (video.isOwned()) {
138 return WEBSERVER.URL + this.getLocalStaticPath()
139 }
140
141 return this.fileUrl
142 }
143
144 getLocalStaticPath () {
145 return LAZY_STATIC_PATHS.STORYBOARDS + this.filename
146 }
147
148 getPath () {
149 return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename)
150 }
151
152 removeFile () {
153 return remove(this.getPath())
154 }
155
156 toFormattedJSON (this: MStoryboardVideo): Storyboard {
157 return {
158 storyboardPath: this.getLocalStaticPath(),
159
160 totalHeight: this.totalHeight,
161 totalWidth: this.totalWidth,
162
163 spriteWidth: this.spriteWidth,
164 spriteHeight: this.spriteHeight,
165
166 spriteDuration: this.spriteDuration
167 }
168 }
169}
diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts
index 1fb1cae82..dd4cefd65 100644
--- a/server/models/video/video-caption.ts
+++ b/server/models/video/video-caption.ts
@@ -15,7 +15,7 @@ import {
15 Table, 15 Table,
16 UpdatedAt 16 UpdatedAt
17} from 'sequelize-typescript' 17} from 'sequelize-typescript'
18import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models' 18import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, MVideoCaptionVideo } from '@server/types/models'
19import { buildUUID } from '@shared/extra-utils' 19import { buildUUID } from '@shared/extra-utils'
20import { AttributesOnly } from '@shared/typescript-utils' 20import { AttributesOnly } from '@shared/typescript-utils'
21import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' 21import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
@@ -225,7 +225,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
225 } 225 }
226 } 226 }
227 227
228 getCaptionStaticPath (this: MVideoCaption) { 228 getCaptionStaticPath (this: MVideoCaptionLanguageUrl) {
229 return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename) 229 return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename)
230 } 230 }
231 231
@@ -233,9 +233,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
233 return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename) 233 return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename)
234 } 234 }
235 235
236 getFileUrl (video: MVideo) { 236 getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideo) {
237 if (!this.Video) this.Video = video as VideoModel
238
239 if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath() 237 if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath()
240 238
241 return this.fileUrl 239 return this.fileUrl
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index f90f2b7f6..0e9a84426 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -58,7 +58,7 @@ import {
58import { AttributesOnly } from '@shared/typescript-utils' 58import { AttributesOnly } from '@shared/typescript-utils'
59import { peertubeTruncate } from '../../helpers/core-utils' 59import { peertubeTruncate } from '../../helpers/core-utils'
60import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 60import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
61import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc' 61import { exists, isArray, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
62import { 62import {
63 isVideoDescriptionValid, 63 isVideoDescriptionValid,
64 isVideoDurationValid, 64 isVideoDurationValid,
@@ -75,6 +75,7 @@ import {
75 MChannel, 75 MChannel,
76 MChannelAccountDefault, 76 MChannelAccountDefault,
77 MChannelId, 77 MChannelId,
78 MStoryboard,
78 MStreamingPlaylist, 79 MStreamingPlaylist,
79 MStreamingPlaylistFilesVideo, 80 MStreamingPlaylistFilesVideo,
80 MUserAccountId, 81 MUserAccountId,
@@ -83,6 +84,8 @@ import {
83 MVideoAccountLight, 84 MVideoAccountLight,
84 MVideoAccountLightBlacklistAllFiles, 85 MVideoAccountLightBlacklistAllFiles,
85 MVideoAP, 86 MVideoAP,
87 MVideoAPLight,
88 MVideoCaptionLanguageUrl,
86 MVideoDetails, 89 MVideoDetails,
87 MVideoFileVideo, 90 MVideoFileVideo,
88 MVideoFormattable, 91 MVideoFormattable,
@@ -126,6 +129,7 @@ import {
126 VideosIdListQueryBuilder, 129 VideosIdListQueryBuilder,
127 VideosModelListQueryBuilder 130 VideosModelListQueryBuilder
128} from './sql/video' 131} from './sql/video'
132import { StoryboardModel } from './storyboard'
129import { TagModel } from './tag' 133import { TagModel } from './tag'
130import { ThumbnailModel } from './thumbnail' 134import { ThumbnailModel } from './thumbnail'
131import { VideoBlacklistModel } from './video-blacklist' 135import { VideoBlacklistModel } from './video-blacklist'
@@ -753,6 +757,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
753 }) 757 })
754 VideoJobInfo: VideoJobInfoModel 758 VideoJobInfo: VideoJobInfoModel
755 759
760 @HasOne(() => StoryboardModel, {
761 foreignKey: {
762 name: 'videoId',
763 allowNull: false
764 },
765 onDelete: 'cascade'
766 })
767 Storyboard: StoryboardModel
768
756 @AfterCreate 769 @AfterCreate
757 static notifyCreate (video: MVideo) { 770 static notifyCreate (video: MVideo) {
758 InternalEventEmitter.Instance.emit('video-created', { video }) 771 InternalEventEmitter.Instance.emit('video-created', { video })
@@ -904,6 +917,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
904 required: false 917 required: false
905 }, 918 },
906 { 919 {
920 model: StoryboardModel.unscoped(),
921 required: false
922 },
923 {
907 attributes: [ 'id', 'url' ], 924 attributes: [ 'id', 'url' ],
908 model: VideoShareModel.unscoped(), 925 model: VideoShareModel.unscoped(),
909 required: false, 926 required: false,
@@ -1768,6 +1785,32 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1768 ) 1785 )
1769 } 1786 }
1770 1787
1788 async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise<MVideoAP> {
1789 const videoAP = this as MVideoAP
1790
1791 const getCaptions = () => {
1792 if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions
1793
1794 return this.$get('VideoCaptions', {
1795 attributes: [ 'filename', 'language', 'fileUrl' ],
1796 transaction
1797 }) as Promise<MVideoCaptionLanguageUrl[]>
1798 }
1799
1800 const getStoryboard = () => {
1801 if (videoAP.Storyboard) return videoAP.Storyboard
1802
1803 return this.$get('Storyboard', { transaction }) as Promise<MStoryboard>
1804 }
1805
1806 const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ])
1807
1808 return Object.assign(this, {
1809 VideoCaptions: captions,
1810 Storyboard: storyboard
1811 })
1812 }
1813
1771 getTruncatedDescription () { 1814 getTruncatedDescription () {
1772 if (!this.description) return null 1815 if (!this.description) return null
1773 1816
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 472cad182..3c752cc3e 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -74,6 +74,9 @@ describe('Test config API validators', function () {
74 }, 74 },
75 torrents: { 75 torrents: {
76 size: 4 76 size: 4
77 },
78 storyboards: {
79 size: 5
77 } 80 }
78 }, 81 },
79 signup: { 82 signup: {
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 400d312d3..c2a7ccd78 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -34,6 +34,7 @@ import './video-comments'
34import './video-files' 34import './video-files'
35import './video-imports' 35import './video-imports'
36import './video-playlists' 36import './video-playlists'
37import './video-storyboards'
37import './video-source' 38import './video-source'
38import './video-studio' 39import './video-studio'
39import './video-token' 40import './video-token'
diff --git a/server/tests/api/check-params/video-storyboards.ts b/server/tests/api/check-params/video-storyboards.ts
new file mode 100644
index 000000000..a43d8fc48
--- /dev/null
+++ b/server/tests/api/check-params/video-storyboards.ts
@@ -0,0 +1,45 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { HttpStatusCode, VideoPrivacy } from '@shared/models'
4import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
5
6describe('Test video storyboards API validator', function () {
7 let server: PeerTubeServer
8
9 let publicVideo: { uuid: string }
10 let privateVideo: { uuid: string }
11
12 // ---------------------------------------------------------------
13
14 before(async function () {
15 this.timeout(30000)
16
17 server = await createSingleServer(1)
18 await setAccessTokensToServers([ server ])
19
20 publicVideo = await server.videos.quickUpload({ name: 'public' })
21 privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE })
22 })
23
24 it('Should fail without a valid uuid', async function () {
25 await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
26 })
27
28 it('Should receive 404 when passing a non existing video id', async function () {
29 await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
30 })
31
32 it('Should not get the private storyboard without the appropriate token', async function () {
33 await server.storyboard.list({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null })
34 await server.storyboard.list({ id: publicVideo.uuid, expectedStatus: HttpStatusCode.OK_200, token: null })
35 })
36
37 it('Should succeed with the correct parameters', async function () {
38 await server.storyboard.list({ id: privateVideo.uuid })
39 await server.storyboard.list({ id: publicVideo.uuid })
40 })
41
42 after(async function () {
43 await cleanupTests([ server ])
44 })
45})
diff --git a/server/tests/api/check-params/videos-overviews.ts b/server/tests/api/check-params/videos-overviews.ts
index f9cdb7ab3..ae7de24dd 100644
--- a/server/tests/api/check-params/videos-overviews.ts
+++ b/server/tests/api/check-params/videos-overviews.ts
@@ -2,7 +2,7 @@
2 2
3import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands' 3import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
4 4
5describe('Test videos overview', function () { 5describe('Test videos overview API validator', function () {
6 let server: PeerTubeServer 6 let server: PeerTubeServer
7 7
8 // --------------------------------------------------------------- 8 // ---------------------------------------------------------------
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index 011ba268c..efa7b50e3 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -46,6 +46,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) {
46 expect(data.cache.previews.size).to.equal(1) 46 expect(data.cache.previews.size).to.equal(1)
47 expect(data.cache.captions.size).to.equal(1) 47 expect(data.cache.captions.size).to.equal(1)
48 expect(data.cache.torrents.size).to.equal(1) 48 expect(data.cache.torrents.size).to.equal(1)
49 expect(data.cache.storyboards.size).to.equal(1)
49 50
50 expect(data.signup.enabled).to.be.true 51 expect(data.signup.enabled).to.be.true
51 expect(data.signup.limit).to.equal(4) 52 expect(data.signup.limit).to.equal(4)
@@ -154,6 +155,7 @@ function checkUpdatedConfig (data: CustomConfig) {
154 expect(data.cache.previews.size).to.equal(2) 155 expect(data.cache.previews.size).to.equal(2)
155 expect(data.cache.captions.size).to.equal(3) 156 expect(data.cache.captions.size).to.equal(3)
156 expect(data.cache.torrents.size).to.equal(4) 157 expect(data.cache.torrents.size).to.equal(4)
158 expect(data.cache.storyboards.size).to.equal(5)
157 159
158 expect(data.signup.enabled).to.be.false 160 expect(data.signup.enabled).to.be.false
159 expect(data.signup.limit).to.equal(5) 161 expect(data.signup.limit).to.equal(5)
@@ -290,6 +292,9 @@ const newCustomConfig: CustomConfig = {
290 }, 292 },
291 torrents: { 293 torrents: {
292 size: 4 294 size: 4
295 },
296 storyboards: {
297 size: 5
293 } 298 }
294 }, 299 },
295 signup: { 300 signup: {
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 357c08199..9c79b3aa6 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -20,3 +20,4 @@ import './videos-history'
20import './videos-overview' 20import './videos-overview'
21import './video-source' 21import './video-source'
22import './video-static-file-privacy' 22import './video-static-file-privacy'
23import './video-storyboard'
diff --git a/server/tests/api/videos/video-storyboard.ts b/server/tests/api/videos/video-storyboard.ts
new file mode 100644
index 000000000..7ccdca8f7
--- /dev/null
+++ b/server/tests/api/videos/video-storyboard.ts
@@ -0,0 +1,184 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { FIXTURE_URLS } from '@server/tests/shared'
5import { areHttpImportTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import {
8 cleanupTests,
9 createMultipleServers,
10 doubleFollow,
11 makeGetRequest,
12 PeerTubeServer,
13 sendRTMPStream,
14 setAccessTokensToServers,
15 setDefaultVideoChannel,
16 stopFfmpeg,
17 waitJobs
18} from '@shared/server-commands'
19
20async function checkStoryboard (options: {
21 server: PeerTubeServer
22 uuid: string
23 tilesCount?: number
24 minSize?: number
25}) {
26 const { server, uuid, tilesCount, minSize = 1000 } = options
27
28 const { storyboards } = await server.storyboard.list({ id: uuid })
29
30 expect(storyboards).to.have.lengthOf(1)
31
32 const storyboard = storyboards[0]
33
34 expect(storyboard.spriteDuration).to.equal(1)
35 expect(storyboard.spriteHeight).to.equal(108)
36 expect(storyboard.spriteWidth).to.equal(192)
37 expect(storyboard.storyboardPath).to.exist
38
39 if (tilesCount) {
40 expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10))
41 expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1))
42 }
43
44 const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
45 expect(body.length).to.be.above(minSize)
46}
47
48describe('Test video storyboard', function () {
49 let servers: PeerTubeServer[]
50
51 before(async function () {
52 this.timeout(120000)
53
54 servers = await createMultipleServers(2)
55 await setAccessTokensToServers(servers)
56 await setDefaultVideoChannel(servers)
57
58 await doubleFollow(servers[0], servers[1])
59 })
60
61 it('Should generate a storyboard after upload without transcoding', async function () {
62 this.timeout(60000)
63
64 // 5s video
65 const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
66 await waitJobs(servers)
67
68 for (const server of servers) {
69 await checkStoryboard({ server, uuid, tilesCount: 5 })
70 }
71 })
72
73 it('Should generate a storyboard after upload without transcoding with a long video', async function () {
74 this.timeout(60000)
75
76 // 124s video
77 const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_very_long_10p.mp4' })
78 await waitJobs(servers)
79
80 for (const server of servers) {
81 await checkStoryboard({ server, uuid, tilesCount: 100 })
82 }
83 })
84
85 it('Should generate a storyboard after upload with transcoding', async function () {
86 this.timeout(60000)
87
88 await servers[0].config.enableMinimumTranscoding()
89
90 // 5s video
91 const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
92 await waitJobs(servers)
93
94 for (const server of servers) {
95 await checkStoryboard({ server, uuid, tilesCount: 5 })
96 }
97 })
98
99 it('Should generate a storyboard after an audio upload', async function () {
100 this.timeout(60000)
101
102 // 6s audio
103 const attributes = { name: 'audio', fixture: 'sample.ogg' }
104 const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' })
105 await waitJobs(servers)
106
107 for (const server of servers) {
108 await checkStoryboard({ server, uuid, tilesCount: 6, minSize: 250 })
109 }
110 })
111
112 it('Should generate a storyboard after HTTP import', async function () {
113 this.timeout(60000)
114
115 if (areHttpImportTestsDisabled()) return
116
117 // 3s video
118 const { video } = await servers[0].imports.importVideo({
119 attributes: {
120 targetUrl: FIXTURE_URLS.goodVideo,
121 channelId: servers[0].store.channel.id,
122 privacy: VideoPrivacy.PUBLIC
123 }
124 })
125 await waitJobs(servers)
126
127 for (const server of servers) {
128 await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 })
129 }
130 })
131
132 it('Should generate a storyboard after torrent import', async function () {
133 this.timeout(60000)
134
135 if (areHttpImportTestsDisabled()) return
136
137 // 10s video
138 const { video } = await servers[0].imports.importVideo({
139 attributes: {
140 magnetUri: FIXTURE_URLS.magnet,
141 channelId: servers[0].store.channel.id,
142 privacy: VideoPrivacy.PUBLIC
143 }
144 })
145 await waitJobs(servers)
146
147 for (const server of servers) {
148 await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 })
149 }
150 })
151
152 it('Should generate a storyboard after a live', async function () {
153 this.timeout(240000)
154
155 await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' })
156
157 const { live, video } = await servers[0].live.quickCreate({
158 saveReplay: true,
159 permanentLive: false,
160 privacy: VideoPrivacy.PUBLIC
161 })
162
163 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
164 await servers[0].live.waitUntilPublished({ videoId: video.id })
165
166 await stopFfmpeg(ffmpegCommand)
167
168 await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id })
169 await waitJobs(servers)
170
171 for (const server of servers) {
172 await checkStoryboard({ server, uuid: video.uuid })
173 }
174 })
175
176 it('Should generate a storyboard with different video durations', async function () {
177 this.timeout(60000)
178
179 })
180
181 after(async function () {
182 await cleanupTests(servers)
183 })
184})
diff --git a/server/tests/fixtures/video_very_long_10p.mp4 b/server/tests/fixtures/video_very_long_10p.mp4
new file mode 100644
index 000000000..852297933
--- /dev/null
+++ b/server/tests/fixtures/video_very_long_10p.mp4
Binary files differ
diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts
index 0ac032290..7f05db666 100644
--- a/server/types/models/video/index.ts
+++ b/server/types/models/video/index.ts
@@ -1,6 +1,7 @@
1export * from './local-video-viewer-watch-section' 1export * from './local-video-viewer-watch-section'
2export * from './local-video-viewer-watch-section' 2export * from './local-video-viewer-watch-section'
3export * from './local-video-viewer' 3export * from './local-video-viewer'
4export * from './storyboard'
4export * from './schedule-video-update' 5export * from './schedule-video-update'
5export * from './tag' 6export * from './tag'
6export * from './thumbnail' 7export * from './thumbnail'
diff --git a/server/types/models/video/storyboard.ts b/server/types/models/video/storyboard.ts
new file mode 100644
index 000000000..a0403d4f0
--- /dev/null
+++ b/server/types/models/video/storyboard.ts
@@ -0,0 +1,15 @@
1import { StoryboardModel } from '@server/models/video/storyboard'
2import { PickWith } from '@shared/typescript-utils'
3import { MVideo } from './video'
4
5type Use<K extends keyof StoryboardModel, M> = PickWith<StoryboardModel, K, M>
6
7// ############################################################################
8
9export type MStoryboard = Omit<StoryboardModel, 'Video'>
10
11// ############################################################################
12
13export type MStoryboardVideo =
14 MStoryboard &
15 Use<'Video', MVideo>
diff --git a/server/types/models/video/video-caption.ts b/server/types/models/video/video-caption.ts
index 8cd801064..d3adec362 100644
--- a/server/types/models/video/video-caption.ts
+++ b/server/types/models/video/video-caption.ts
@@ -11,7 +11,7 @@ export type MVideoCaption = Omit<VideoCaptionModel, 'Video'>
11// ############################################################################ 11// ############################################################################
12 12
13export type MVideoCaptionLanguage = Pick<MVideoCaption, 'language'> 13export type MVideoCaptionLanguage = Pick<MVideoCaption, 'language'>
14export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'getFileUrl'> 14export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'filename' | 'getFileUrl' | 'getCaptionStaticPath'>
15 15
16export type MVideoCaptionVideo = 16export type MVideoCaptionVideo =
17 MVideoCaption & 17 MVideoCaption &
diff --git a/server/types/models/video/video.ts b/server/types/models/video/video.ts
index 8021e56bb..53ee94269 100644
--- a/server/types/models/video/video.ts
+++ b/server/types/models/video/video.ts
@@ -3,6 +3,7 @@ import { VideoModel } from '../../../models/video/video'
3import { MTrackerUrl } from '../server/tracker' 3import { MTrackerUrl } from '../server/tracker'
4import { MUserVideoHistoryTime } from '../user/user-video-history' 4import { MUserVideoHistoryTime } from '../user/user-video-history'
5import { MScheduleVideoUpdate } from './schedule-video-update' 5import { MScheduleVideoUpdate } from './schedule-video-update'
6import { MStoryboard } from './storyboard'
6import { MTag } from './tag' 7import { MTag } from './tag'
7import { MThumbnail } from './thumbnail' 8import { MThumbnail } from './thumbnail'
8import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' 9import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist'
@@ -32,7 +33,7 @@ type Use<K extends keyof VideoModel, M> = PickWith<VideoModel, K, M>
32export type MVideo = 33export type MVideo =
33 Omit<VideoModel, 'VideoChannel' | 'Tags' | 'Thumbnails' | 'VideoPlaylistElements' | 'VideoAbuses' | 34 Omit<VideoModel, 'VideoChannel' | 'Tags' | 'Thumbnails' | 'VideoPlaylistElements' | 'VideoAbuses' |
34 'VideoFiles' | 'VideoStreamingPlaylists' | 'VideoShares' | 'AccountVideoRates' | 'VideoComments' | 'VideoViews' | 'UserVideoHistories' | 35 'VideoFiles' | 'VideoStreamingPlaylists' | 'VideoShares' | 'AccountVideoRates' | 'VideoComments' | 'VideoViews' | 'UserVideoHistories' |
35 'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords'> 36 'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords' | 'Storyboard'>
36 37
37// ############################################################################ 38// ############################################################################
38 39
@@ -173,9 +174,10 @@ export type MVideoAP =
173 Use<'VideoBlacklist', MVideoBlacklistUnfederated> & 174 Use<'VideoBlacklist', MVideoBlacklistUnfederated> &
174 Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & 175 Use<'VideoFiles', MVideoFileRedundanciesOpt[]> &
175 Use<'Thumbnails', MThumbnail[]> & 176 Use<'Thumbnails', MThumbnail[]> &
176 Use<'VideoLive', MVideoLive> 177 Use<'VideoLive', MVideoLive> &
178 Use<'Storyboard', MStoryboard>
177 179
178export type MVideoAPWithoutCaption = Omit<MVideoAP, 'VideoCaptions'> 180export type MVideoAPLight = Omit<MVideoAP, 'VideoCaptions' | 'Storyboard'>
179 181
180export type MVideoDetails = 182export type MVideoDetails =
181 MVideo & 183 MVideo &
diff --git a/shared/ffmpeg/ffmpeg-images.ts b/shared/ffmpeg/ffmpeg-images.ts
index 2db63bd8b..27305382c 100644
--- a/shared/ffmpeg/ffmpeg-images.ts
+++ b/shared/ffmpeg/ffmpeg-images.ts
@@ -56,4 +56,41 @@ export class FFmpegImage {
56 .thumbnail(thumbnailOptions) 56 .thumbnail(thumbnailOptions)
57 }) 57 })
58 } 58 }
59
60 async generateStoryboardFromVideo (options: {
61 path: string
62 destination: string
63
64 sprites: {
65 size: {
66 width: number
67 height: number
68 }
69
70 count: {
71 width: number
72 height: number
73 }
74
75 duration: number
76 }
77 }) {
78 const { path, destination, sprites } = options
79
80 const command = this.commandWrapper.buildCommand(path)
81
82 const filter = [
83 `setpts=N/round(FRAME_RATE)/TB`,
84 `select='not(mod(t,${options.sprites.duration}))'`,
85 `scale=${sprites.size.width}:${sprites.size.height}`,
86 `tile=layout=${sprites.count.width}x${sprites.count.height}`
87 ].join(',')
88
89 command.outputOption('-filter_complex', filter)
90 command.outputOption('-frames:v', '1')
91 command.outputOption('-q:v', '2')
92 command.output(destination)
93
94 return this.commandWrapper.runCommand()
95 }
59} 96}
diff --git a/shared/models/activitypub/objects/index.ts b/shared/models/activitypub/objects/index.ts
index 9aa3c462c..a2e040b32 100644
--- a/shared/models/activitypub/objects/index.ts
+++ b/shared/models/activitypub/objects/index.ts
@@ -6,5 +6,5 @@ export * from './object.model'
6export * from './playlist-element-object' 6export * from './playlist-element-object'
7export * from './playlist-object' 7export * from './playlist-object'
8export * from './video-comment-object' 8export * from './video-comment-object'
9export * from './video-torrent-object' 9export * from './video-object'
10export * from './watch-action-object' 10export * from './watch-action-object'
diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-object.ts
index 23d54bdbd..a252a2df0 100644
--- a/shared/models/activitypub/objects/video-torrent-object.ts
+++ b/shared/models/activitypub/objects/video-object.ts
@@ -51,6 +51,22 @@ export interface VideoObject {
51 51
52 attributedTo: ActivityPubAttributedTo[] 52 attributedTo: ActivityPubAttributedTo[]
53 53
54 preview?: ActivityPubStoryboard[]
55
54 to?: string[] 56 to?: string[]
55 cc?: string[] 57 cc?: string[]
56} 58}
59
60export interface ActivityPubStoryboard {
61 type: 'Image'
62 rel: [ 'storyboard' ]
63 url: {
64 href: string
65 mediaType: string
66 width: number
67 height: number
68 tileWidth: number
69 tileHeight: number
70 tileDuration: string
71 }[]
72}
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts
index 4202589f3..1012312f3 100644
--- a/shared/models/server/custom-config.model.ts
+++ b/shared/models/server/custom-config.model.ts
@@ -78,6 +78,10 @@ export interface CustomConfig {
78 torrents: { 78 torrents: {
79 size: number 79 size: number
80 } 80 }
81
82 storyboards: {
83 size: number
84 }
81 } 85 }
82 86
83 signup: { 87 signup: {
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 22ecee324..9c40079fb 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -30,6 +30,7 @@ export type JobType =
30 | 'video-studio-edition' 30 | 'video-studio-edition'
31 | 'video-transcoding' 31 | 'video-transcoding'
32 | 'videos-views-stats' 32 | 'videos-views-stats'
33 | 'generate-video-storyboard'
33 34
34export interface Job { 35export interface Job {
35 id: number | string 36 id: number | string
@@ -294,3 +295,10 @@ export interface TranscodingJobBuilderPayload {
294 priority?: number 295 priority?: number
295 }[][] 296 }[][]
296} 297}
298
299// ---------------------------------------------------------------------------
300
301export interface GenerateStoryboardPayload {
302 videoUUID: string
303 federate: boolean
304}
diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts
index 80be1854b..b3ce6ad3f 100644
--- a/shared/models/videos/index.ts
+++ b/shared/models/videos/index.ts
@@ -15,6 +15,7 @@ export * from './channel-sync'
15 15
16export * from './nsfw-policy.type' 16export * from './nsfw-policy.type'
17 17
18export * from './storyboard.model'
18export * from './thumbnail.type' 19export * from './thumbnail.type'
19 20
20export * from './video-constant.model' 21export * from './video-constant.model'
diff --git a/shared/models/videos/storyboard.model.ts b/shared/models/videos/storyboard.model.ts
new file mode 100644
index 000000000..c92c81f09
--- /dev/null
+++ b/shared/models/videos/storyboard.model.ts
@@ -0,0 +1,11 @@
1export interface Storyboard {
2 storyboardPath: string
3
4 totalHeight: number
5 totalWidth: number
6
7 spriteHeight: number
8 spriteWidth: number
9
10 spriteDuration: number
11}
diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts
index b94bd2625..114db8091 100644
--- a/shared/server-commands/server/config-command.ts
+++ b/shared/server-commands/server/config-command.ts
@@ -159,6 +159,10 @@ export class ConfigCommand extends AbstractCommand {
159 newConfig: { 159 newConfig: {
160 transcoding: { 160 transcoding: {
161 enabled: true, 161 enabled: true,
162
163 allowAudioFiles: true,
164 allowAdditionalExtensions: true,
165
162 resolutions: { 166 resolutions: {
163 ...ConfigCommand.getCustomConfigResolutions(false), 167 ...ConfigCommand.getCustomConfigResolutions(false),
164 168
@@ -368,6 +372,9 @@ export class ConfigCommand extends AbstractCommand {
368 }, 372 },
369 torrents: { 373 torrents: {
370 size: 4 374 size: 4
375 },
376 storyboards: {
377 size: 5
371 } 378 }
372 }, 379 },
373 signup: { 380 signup: {
diff --git a/shared/server-commands/server/jobs.ts b/shared/server-commands/server/jobs.ts
index ff3098063..8f131fba4 100644
--- a/shared/server-commands/server/jobs.ts
+++ b/shared/server-commands/server/jobs.ts
@@ -33,6 +33,8 @@ async function waitJobs (
33 33
34 // Check if each server has pending request 34 // Check if each server has pending request
35 for (const server of servers) { 35 for (const server of servers) {
36 if (process.env.DEBUG) console.log('Checking ' + server.url)
37
36 for (const state of states) { 38 for (const state of states) {
37 39
38 const jobPromise = server.jobs.list({ 40 const jobPromise = server.jobs.list({
@@ -45,6 +47,10 @@ async function waitJobs (
45 .then(jobs => { 47 .then(jobs => {
46 if (jobs.length !== 0) { 48 if (jobs.length !== 0) {
47 pendingRequests = true 49 pendingRequests = true
50
51 if (process.env.DEBUG) {
52 console.log(jobs)
53 }
48 } 54 }
49 }) 55 })
50 56
@@ -55,6 +61,10 @@ async function waitJobs (
55 .then(obj => { 61 .then(obj => {
56 if (obj.activityPubMessagesWaiting !== 0) { 62 if (obj.activityPubMessagesWaiting !== 0) {
57 pendingRequests = true 63 pendingRequests = true
64
65 if (process.env.DEBUG) {
66 console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting)
67 }
58 } 68 }
59 }) 69 })
60 tasks.push(debugPromise) 70 tasks.push(debugPromise)
@@ -65,12 +75,15 @@ async function waitJobs (
65 for (const job of data) { 75 for (const job of data) {
66 if (job.state.id !== RunnerJobState.COMPLETED) { 76 if (job.state.id !== RunnerJobState.COMPLETED) {
67 pendingRequests = true 77 pendingRequests = true
78
79 if (process.env.DEBUG) {
80 console.log(job)
81 }
68 } 82 }
69 } 83 }
70 }) 84 })
71 tasks.push(runnerJobsPromise) 85 tasks.push(runnerJobsPromise)
72 } 86 }
73
74 } 87 }
75 88
76 return tasks 89 return tasks
diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index 0911e22b0..6aa4296b0 100644
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -35,6 +35,7 @@ import {
35 VideoPasswordsCommand, 35 VideoPasswordsCommand,
36 PlaylistsCommand, 36 PlaylistsCommand,
37 ServicesCommand, 37 ServicesCommand,
38 StoryboardCommand,
38 StreamingPlaylistsCommand, 39 StreamingPlaylistsCommand,
39 VideosCommand, 40 VideosCommand,
40 VideoStudioCommand, 41 VideoStudioCommand,
@@ -149,6 +150,8 @@ export class PeerTubeServer {
149 registrations?: RegistrationsCommand 150 registrations?: RegistrationsCommand
150 videoPasswords?: VideoPasswordsCommand 151 videoPasswords?: VideoPasswordsCommand
151 152
153 storyboard?: StoryboardCommand
154
152 runners?: RunnersCommand 155 runners?: RunnersCommand
153 runnerRegistrationTokens?: RunnerRegistrationTokensCommand 156 runnerRegistrationTokens?: RunnerRegistrationTokensCommand
154 runnerJobs?: RunnerJobsCommand 157 runnerJobs?: RunnerJobsCommand
@@ -436,6 +439,8 @@ export class PeerTubeServer {
436 this.videoToken = new VideoTokenCommand(this) 439 this.videoToken = new VideoTokenCommand(this)
437 this.registrations = new RegistrationsCommand(this) 440 this.registrations = new RegistrationsCommand(this)
438 441
442 this.storyboard = new StoryboardCommand(this)
443
439 this.runners = new RunnersCommand(this) 444 this.runners = new RunnersCommand(this)
440 this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this) 445 this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this)
441 this.runnerJobs = new RunnerJobsCommand(this) 446 this.runnerJobs = new RunnerJobsCommand(this)
diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts
index da36b5b6b..106d80af0 100644
--- a/shared/server-commands/videos/index.ts
+++ b/shared/server-commands/videos/index.ts
@@ -11,6 +11,7 @@ export * from './live-command'
11export * from './live' 11export * from './live'
12export * from './playlists-command' 12export * from './playlists-command'
13export * from './services-command' 13export * from './services-command'
14export * from './storyboard-command'
14export * from './streaming-playlists-command' 15export * from './streaming-playlists-command'
15export * from './comments-command' 16export * from './comments-command'
16export * from './video-studio-command' 17export * from './video-studio-command'
diff --git a/shared/server-commands/videos/storyboard-command.ts b/shared/server-commands/videos/storyboard-command.ts
new file mode 100644
index 000000000..06d90fc12
--- /dev/null
+++ b/shared/server-commands/videos/storyboard-command.ts
@@ -0,0 +1,19 @@
1import { HttpStatusCode, Storyboard } from '@shared/models'
2import { AbstractCommand, OverrideCommandOptions } from '../shared'
3
4export class StoryboardCommand extends AbstractCommand {
5
6 list (options: OverrideCommandOptions & {
7 id: number | string
8 }) {
9 const path = '/api/v1/videos/' + options.id + '/storyboards'
10
11 return this.getRequestBody<{ storyboards: Storyboard[] }>({
12 ...options,
13
14 path,
15 implicitToken: true,
16 defaultExpectedStatus: HttpStatusCode.OK_200
17 })
18 }
19}
diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml
index ff94f802b..cd0e6ffd8 100644
--- a/support/doc/api/openapi.yaml
+++ b/support/doc/api/openapi.yaml
@@ -3668,6 +3668,27 @@ paths:
3668 items: 3668 items:
3669 $ref: '#/components/schemas/VideoBlacklist' 3669 $ref: '#/components/schemas/VideoBlacklist'
3670 3670
3671 /api/v1/videos/{id}/storyboards:
3672 get:
3673 summary: List storyboards of a video
3674 operationId: listVideoStoryboards
3675 tags:
3676 - Video
3677 parameters:
3678 - $ref: '#/components/parameters/idOrUUID'
3679 responses:
3680 '200':
3681 description: successful operation
3682 content:
3683 application/json:
3684 schema:
3685 type: object
3686 properties:
3687 storyboards:
3688 type: array
3689 items:
3690 $ref: '#/components/schemas/Storyboard'
3691
3671 /api/v1/videos/{id}/captions: 3692 /api/v1/videos/{id}/captions:
3672 get: 3693 get:
3673 summary: List captions of a video 3694 summary: List captions of a video
@@ -7509,6 +7530,20 @@ components:
7509 type: array 7530 type: array
7510 items: 7531 items:
7511 $ref: '#/components/schemas/VideoCommentThreadTree' 7532 $ref: '#/components/schemas/VideoCommentThreadTree'
7533 Storyboard:
7534 properties:
7535 storyboardPath:
7536 type: string
7537 totalHeight:
7538 type: integer
7539 totalWidth:
7540 type: integer
7541 spriteHeight:
7542 type: integer
7543 spriteWidth:
7544 type: integer
7545 spriteDuration:
7546 type: integer
7512 VideoCaption: 7547 VideoCaption:
7513 properties: 7548 properties:
7514 language: 7549 language: