aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/activitypub/client.ts10
-rw-r--r--server/controllers/api/config.ts3
-rw-r--r--server/controllers/api/videos/index.ts2
-rw-r--r--server/controllers/api/videos/storyboard.ts29
-rw-r--r--server/controllers/api/videos/upload.ts9
-rw-r--r--server/controllers/lazy-static.ts15
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts39
-rw-r--r--server/initializers/checker-before-init.ts3
-rw-r--r--server/initializers/config.ts4
-rw-r--r--server/initializers/constants.ts19
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/lib/activitypub/context.ts13
-rw-r--r--server/lib/activitypub/send/send-update.ts14
-rw-r--r--server/lib/activitypub/videos/federate.ts13
-rw-r--r--server/lib/activitypub/videos/shared/abstract-builder.ts12
-rw-r--r--server/lib/activitypub/videos/shared/creator.ts1
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts26
-rw-r--r--server/lib/activitypub/videos/updater.ts5
-rw-r--r--server/lib/files-cache/index.ts3
-rw-r--r--server/lib/files-cache/videos-storyboard-cache.ts53
-rw-r--r--server/lib/job-queue/handlers/generate-storyboard.ts138
-rw-r--r--server/lib/job-queue/handlers/video-import.ts9
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts11
-rw-r--r--server/lib/redis.ts4
-rw-r--r--server/lib/transcoding/web-transcoding.ts18
-rw-r--r--server/middlewares/validators/config.ts1
-rw-r--r--server/models/video/formatter/video-format-utils.ts60
-rw-r--r--server/models/video/storyboard.ts169
-rw-r--r--server/models/video/video-caption.ts8
-rw-r--r--server/models/video/video.ts45
-rw-r--r--server/tests/api/check-params/config.ts3
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/video-storyboards.ts45
-rw-r--r--server/tests/api/check-params/videos-overviews.ts2
-rw-r--r--server/tests/api/server/config.ts5
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/video-storyboard.ts184
-rw-r--r--server/tests/fixtures/video_very_long_10p.mp4bin0 -> 185338 bytes
-rw-r--r--server/types/models/video/index.ts1
-rw-r--r--server/types/models/video/storyboard.ts15
-rw-r--r--server/types/models/video/video-caption.ts2
-rw-r--r--server/types/models/video/video.ts8
43 files changed, 953 insertions, 74 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 750e3091c..166fc2a22 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -33,7 +33,6 @@ import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '
33import { AccountModel } from '../../models/account/account' 33import { AccountModel } from '../../models/account/account'
34import { AccountVideoRateModel } from '../../models/account/account-video-rate' 34import { AccountVideoRateModel } from '../../models/account/account-video-rate'
35import { ActorFollowModel } from '../../models/actor/actor-follow' 35import { ActorFollowModel } from '../../models/actor/actor-follow'
36import { VideoCaptionModel } from '../../models/video/video-caption'
37import { VideoCommentModel } from '../../models/video/video-comment' 36import { VideoCommentModel } from '../../models/video/video-comment'
38import { VideoPlaylistModel } from '../../models/video/video-playlist' 37import { VideoPlaylistModel } from '../../models/video/video-playlist'
39import { VideoShareModel } from '../../models/video/video-share' 38import { VideoShareModel } from '../../models/video/video-share'
@@ -242,14 +241,13 @@ async function videoController (req: express.Request, res: express.Response) {
242 if (redirectIfNotOwned(video.url, res)) return 241 if (redirectIfNotOwned(video.url, res)) return
243 242
244 // We need captions to render AP object 243 // We need captions to render AP object
245 const captions = await VideoCaptionModel.listVideoCaptions(video.id) 244 const videoAP = await video.lightAPToFullAP(undefined)
246 const videoWithCaptions = Object.assign(video, { VideoCaptions: captions })
247 245
248 const audience = getAudience(videoWithCaptions.VideoChannel.Account.Actor, videoWithCaptions.privacy === VideoPrivacy.PUBLIC) 246 const audience = getAudience(videoAP.VideoChannel.Account.Actor, videoAP.privacy === VideoPrivacy.PUBLIC)
249 const videoObject = audiencify(await videoWithCaptions.toActivityPubObject(), audience) 247 const videoObject = audiencify(await videoAP.toActivityPubObject(), audience)
250 248
251 if (req.path.endsWith('/activity')) { 249 if (req.path.endsWith('/activity')) {
252 const data = buildCreateActivity(videoWithCaptions.url, video.VideoChannel.Account.Actor, videoObject, audience) 250 const data = buildCreateActivity(videoAP.url, video.VideoChannel.Account.Actor, videoObject, audience)
253 return activityPubResponse(activityPubContextify(data, 'Video'), res) 251 return activityPubResponse(activityPubContextify(data, 'Video'), res)
254 } 252 }
255 253
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 228eae109..c1f6756de 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -190,6 +190,9 @@ function customConfig (): CustomConfig {
190 }, 190 },
191 torrents: { 191 torrents: {
192 size: CONFIG.CACHE.TORRENTS.SIZE 192 size: CONFIG.CACHE.TORRENTS.SIZE
193 },
194 storyboards: {
195 size: CONFIG.CACHE.STORYBOARDS.SIZE
193 } 196 }
194 }, 197 },
195 signup: { 198 signup: {
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index d0eecf812..bbdda5b29 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -41,6 +41,7 @@ import { liveRouter } from './live'
41import { ownershipVideoRouter } from './ownership' 41import { ownershipVideoRouter } from './ownership'
42import { rateVideoRouter } from './rate' 42import { rateVideoRouter } from './rate'
43import { statsRouter } from './stats' 43import { statsRouter } from './stats'
44import { storyboardRouter } from './storyboard'
44import { studioRouter } from './studio' 45import { studioRouter } from './studio'
45import { tokenRouter } from './token' 46import { tokenRouter } from './token'
46import { transcodingRouter } from './transcoding' 47import { transcodingRouter } from './transcoding'
@@ -70,6 +71,7 @@ videosRouter.use('/', filesRouter)
70videosRouter.use('/', transcodingRouter) 71videosRouter.use('/', transcodingRouter)
71videosRouter.use('/', tokenRouter) 72videosRouter.use('/', tokenRouter)
72videosRouter.use('/', videoPasswordRouter) 73videosRouter.use('/', videoPasswordRouter)
74videosRouter.use('/', storyboardRouter)
73 75
74videosRouter.get('/categories', 76videosRouter.get('/categories',
75 openapiOperationDoc({ operationId: 'getCategories' }), 77 openapiOperationDoc({ operationId: 'getCategories' }),
diff --git a/server/controllers/api/videos/storyboard.ts b/server/controllers/api/videos/storyboard.ts
new file mode 100644
index 000000000..47a22011d
--- /dev/null
+++ b/server/controllers/api/videos/storyboard.ts
@@ -0,0 +1,29 @@
1import express from 'express'
2import { getVideoWithAttributes } from '@server/helpers/video'
3import { StoryboardModel } from '@server/models/video/storyboard'
4import { asyncMiddleware, videosGetValidator } from '../../../middlewares'
5
6const storyboardRouter = express.Router()
7
8storyboardRouter.get('/:id/storyboards',
9 asyncMiddleware(videosGetValidator),
10 asyncMiddleware(listStoryboards)
11)
12
13// ---------------------------------------------------------------------------
14
15export {
16 storyboardRouter
17}
18
19// ---------------------------------------------------------------------------
20
21async function listStoryboards (req: express.Request, res: express.Response) {
22 const video = getVideoWithAttributes(res)
23
24 const storyboards = await StoryboardModel.listStoryboardsOf(video)
25
26 return res.json({
27 storyboards: storyboards.map(s => s.toFormattedJSON())
28 })
29}
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 073eb480f..86ab4591e 100644
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -235,6 +235,15 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
235 }, 235 },
236 236
237 { 237 {
238 type: 'generate-video-storyboard' as 'generate-video-storyboard',
239 payload: {
240 videoUUID: video.uuid,
241 // No need to federate, we process these jobs sequentially
242 federate: false
243 }
244 },
245
246 {
238 type: 'notify', 247 type: 'notify',
239 payload: { 248 payload: {
240 action: 'new-video', 249 action: 'new-video',
diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts
index b082e41f6..6ffd39730 100644
--- a/server/controllers/lazy-static.ts
+++ b/server/controllers/lazy-static.ts
@@ -5,7 +5,7 @@ import { MActorImage } from '@server/types/models'
5import { HttpStatusCode } from '../../shared/models/http/http-error-codes' 5import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
6import { logger } from '../helpers/logger' 6import { logger } from '../helpers/logger'
7import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants' 7import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
8import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache' 8import { VideosCaptionCache, VideosPreviewCache, VideosStoryboardCache } from '../lib/files-cache'
9import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor' 9import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor'
10import { asyncMiddleware, handleStaticError } from '../middlewares' 10import { asyncMiddleware, handleStaticError } from '../middlewares'
11import { ActorImageModel } from '../models/actor/actor-image' 11import { ActorImageModel } from '../models/actor/actor-image'
@@ -33,6 +33,12 @@ lazyStaticRouter.use(
33) 33)
34 34
35lazyStaticRouter.use( 35lazyStaticRouter.use(
36 LAZY_STATIC_PATHS.STORYBOARDS + ':filename',
37 asyncMiddleware(getStoryboard),
38 handleStaticError
39)
40
41lazyStaticRouter.use(
36 LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename', 42 LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename',
37 asyncMiddleware(getVideoCaption), 43 asyncMiddleware(getVideoCaption),
38 handleStaticError 44 handleStaticError
@@ -126,6 +132,13 @@ async function getPreview (req: express.Request, res: express.Response) {
126 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) 132 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
127} 133}
128 134
135async function getStoryboard (req: express.Request, res: express.Response) {
136 const result = await VideosStoryboardCache.Instance.getFilePath(req.params.filename)
137 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
138
139 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
140}
141
129async function getVideoCaption (req: express.Request, res: express.Response) { 142async function getVideoCaption (req: express.Request, res: express.Response) {
130 const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename) 143 const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename)
131 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() 144 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 97b3577af..573a29754 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -1,6 +1,6 @@
1import validator from 'validator' 1import validator from 'validator'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@shared/models' 3import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models'
4import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' 4import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos'
5import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' 5import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
6import { peertubeTruncate } from '../../core-utils' 6import { peertubeTruncate } from '../../core-utils'
@@ -48,6 +48,10 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
48 logger.debug('Video has invalid icons', { video }) 48 logger.debug('Video has invalid icons', { video })
49 return false 49 return false
50 } 50 }
51 if (!setValidStoryboard(video)) {
52 logger.debug('Video has invalid preview (storyboard)', { video })
53 return false
54 }
51 55
52 // Default attributes 56 // Default attributes
53 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED 57 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
@@ -201,3 +205,36 @@ function setRemoteVideoContent (video: any) {
201 205
202 return true 206 return true
203} 207}
208
209function setValidStoryboard (video: VideoObject) {
210 if (!video.preview) return true
211 if (!Array.isArray(video.preview)) return false
212
213 video.preview = video.preview.filter(p => isStorybordValid(p))
214
215 return true
216}
217
218function isStorybordValid (preview: ActivityPubStoryboard) {
219 if (!preview) return false
220
221 if (
222 preview.type !== 'Image' ||
223 !isArray(preview.rel) ||
224 !preview.rel.includes('storyboard')
225 ) {
226 return false
227 }
228
229 preview.url = preview.url.filter(u => {
230 return u.mediaType === 'image/jpeg' &&
231 isActivityPubUrlValid(u.href) &&
232 validator.isInt(u.width + '', { min: 0 }) &&
233 validator.isInt(u.height + '', { min: 0 }) &&
234 validator.isInt(u.tileWidth + '', { min: 0 }) &&
235 validator.isInt(u.tileHeight + '', { min: 0 }) &&
236 isActivityPubVideoDurationValid(u.tileDuration)
237 })
238
239 return preview.url.length !== 0
240}
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 0a315ea70..939b73344 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -29,7 +29,8 @@ function checkMissedConfig () {
29 'video_channels.max_per_user', 29 'video_channels.max_per_user',
30 'csp.enabled', 'csp.report_only', 'csp.report_uri', 30 'csp.enabled', 'csp.report_only', 'csp.report_uri',
31 'security.frameguard.enabled', 'security.powered_by_header.enabled', 31 'security.frameguard.enabled', 'security.powered_by_header.enabled',
32 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'admin.email', 'contact_form.enabled', 32 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'cache.storyboards.size',
33 'admin.email', 'contact_form.enabled',
33 'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age', 34 'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age',
34 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 35 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
35 'redundancy.videos.strategies', 'redundancy.videos.check_interval', 36 'redundancy.videos.strategies', 'redundancy.videos.check_interval',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 51ac5d0ce..60ab6e204 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -112,6 +112,7 @@ const CONFIG = {
112 STREAMING_PLAYLISTS_DIR: buildPath(config.get<string>('storage.streaming_playlists')), 112 STREAMING_PLAYLISTS_DIR: buildPath(config.get<string>('storage.streaming_playlists')),
113 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), 113 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
114 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), 114 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
115 STORYBOARDS_DIR: buildPath(config.get<string>('storage.storyboards')),
115 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), 116 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
116 CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')), 117 CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),
117 TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')), 118 TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
@@ -482,6 +483,9 @@ const CONFIG = {
482 }, 483 },
483 TORRENTS: { 484 TORRENTS: {
484 get SIZE () { return config.get<number>('cache.torrents.size') } 485 get SIZE () { return config.get<number>('cache.torrents.size') }
486 },
487 STORYBOARDS: {
488 get SIZE () { return config.get<number>('cache.storyboards.size') }
485 } 489 }
486 }, 490 },
487 INSTANCE: { 491 INSTANCE: {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index e2f34fe16..3a643a60b 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -174,6 +174,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
174 'after-video-channel-import': 1, 174 'after-video-channel-import': 1,
175 'move-to-object-storage': 3, 175 'move-to-object-storage': 3,
176 'transcoding-job-builder': 1, 176 'transcoding-job-builder': 1,
177 'generate-video-storyboard': 1,
177 'notify': 1, 178 'notify': 1,
178 'federate-video': 1 179 'federate-video': 1
179} 180}
@@ -198,6 +199,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
198 'video-channel-import': 1, 199 'video-channel-import': 1,
199 'after-video-channel-import': 1, 200 'after-video-channel-import': 1,
200 'transcoding-job-builder': 1, 201 'transcoding-job-builder': 1,
202 'generate-video-storyboard': 1,
201 'notify': 5, 203 'notify': 5,
202 'federate-video': 3 204 'federate-video': 3
203} 205}
@@ -218,6 +220,7 @@ const JOB_TTL: { [id in JobType]: number } = {
218 'activitypub-refresher': 60000 * 10, // 10 minutes 220 'activitypub-refresher': 60000 * 10, // 10 minutes
219 'video-redundancy': 1000 * 3600 * 3, // 3 hours 221 'video-redundancy': 1000 * 3600 * 3, // 3 hours
220 'video-live-ending': 1000 * 60 * 10, // 10 minutes 222 'video-live-ending': 1000 * 60 * 10, // 10 minutes
223 'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes
221 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours 224 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
222 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours 225 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
223 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours 226 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
@@ -766,7 +769,8 @@ const LAZY_STATIC_PATHS = {
766 AVATARS: '/lazy-static/avatars/', 769 AVATARS: '/lazy-static/avatars/',
767 PREVIEWS: '/lazy-static/previews/', 770 PREVIEWS: '/lazy-static/previews/',
768 VIDEO_CAPTIONS: '/lazy-static/video-captions/', 771 VIDEO_CAPTIONS: '/lazy-static/video-captions/',
769 TORRENTS: '/lazy-static/torrents/' 772 TORRENTS: '/lazy-static/torrents/',
773 STORYBOARDS: '/lazy-static/storyboards/'
770} 774}
771const OBJECT_STORAGE_PROXY_PATHS = { 775const OBJECT_STORAGE_PROXY_PATHS = {
772 PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/', 776 PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/',
@@ -813,6 +817,14 @@ const ACTOR_IMAGES_SIZE: { [key in ActorImageType]: { width: number, height: num
813 ] 817 ]
814} 818}
815 819
820const STORYBOARD = {
821 SPRITE_SIZE: {
822 width: 192,
823 height: 108
824 },
825 SPRITES_MAX_EDGE_COUNT: 10
826}
827
816const EMBED_SIZE = { 828const EMBED_SIZE = {
817 width: 560, 829 width: 560,
818 height: 315 830 height: 315
@@ -824,6 +836,10 @@ const FILES_CACHE = {
824 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'), 836 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
825 MAX_AGE: 1000 * 3600 * 3 // 3 hours 837 MAX_AGE: 1000 * 3600 * 3 // 3 hours
826 }, 838 },
839 STORYBOARDS: {
840 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'storyboards'),
841 MAX_AGE: 1000 * 3600 * 24 // 24 hours
842 },
827 VIDEO_CAPTIONS: { 843 VIDEO_CAPTIONS: {
828 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'), 844 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'),
829 MAX_AGE: 1000 * 3600 * 3 // 3 hours 845 MAX_AGE: 1000 * 3600 * 3 // 3 hours
@@ -1090,6 +1106,7 @@ export {
1090 RESUMABLE_UPLOAD_SESSION_LIFETIME, 1106 RESUMABLE_UPLOAD_SESSION_LIFETIME,
1091 RUNNER_JOB_STATES, 1107 RUNNER_JOB_STATES,
1092 P2P_MEDIA_LOADER_PEER_VERSION, 1108 P2P_MEDIA_LOADER_PEER_VERSION,
1109 STORYBOARD,
1093 ACTOR_IMAGES_SIZE, 1110 ACTOR_IMAGES_SIZE,
1094 ACCEPT_HEADERS, 1111 ACCEPT_HEADERS,
1095 BCRYPT_SALT_SIZE, 1112 BCRYPT_SALT_SIZE,
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 9e926c26c..bc120e398 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -10,6 +10,7 @@ import { UserModel } from '@server/models/user/user'
10import { UserNotificationModel } from '@server/models/user/user-notification' 10import { UserNotificationModel } from '@server/models/user/user-notification'
11import { UserRegistrationModel } from '@server/models/user/user-registration' 11import { UserRegistrationModel } from '@server/models/user/user-registration'
12import { UserVideoHistoryModel } from '@server/models/user/user-video-history' 12import { UserVideoHistoryModel } from '@server/models/user/user-video-history'
13import { StoryboardModel } from '@server/models/video/storyboard'
13import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 14import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
14import { VideoJobInfoModel } from '@server/models/video/video-job-info' 15import { VideoJobInfoModel } from '@server/models/video/video-job-info'
15import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' 16import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
@@ -167,7 +168,8 @@ async function initDatabaseModels (silent: boolean) {
167 VideoPasswordModel, 168 VideoPasswordModel,
168 RunnerRegistrationTokenModel, 169 RunnerRegistrationTokenModel,
169 RunnerModel, 170 RunnerModel,
170 RunnerJobModel 171 RunnerJobModel,
172 StoryboardModel
171 ]) 173 ])
172 174
173 // Check extensions exist in the database 175 // Check extensions exist in the database
diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts
index a3ca52a31..750276a11 100644
--- a/server/lib/activitypub/context.ts
+++ b/server/lib/activitypub/context.ts
@@ -46,6 +46,19 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string
46 46
47 Infohash: 'pt:Infohash', 47 Infohash: 'pt:Infohash',
48 48
49 tileWidth: {
50 '@type': 'sc:Number',
51 '@id': 'pt:tileWidth'
52 },
53 tileHeight: {
54 '@type': 'sc:Number',
55 '@id': 'pt:tileHeight'
56 },
57 tileDuration: {
58 '@type': 'sc:Number',
59 '@id': 'pt:tileDuration'
60 },
61
49 originallyPublishedAt: 'sc:datePublished', 62 originallyPublishedAt: 'sc:datePublished',
50 views: { 63 views: {
51 '@type': 'sc:Number', 64 '@type': 'sc:Number',
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index 379e2d9d8..3d2b437e4 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -10,8 +10,7 @@ import {
10 MActor, 10 MActor,
11 MActorLight, 11 MActorLight,
12 MChannelDefault, 12 MChannelDefault,
13 MVideoAP, 13 MVideoAPLight,
14 MVideoAPWithoutCaption,
15 MVideoPlaylistFull, 14 MVideoPlaylistFull,
16 MVideoRedundancyVideo 15 MVideoRedundancyVideo
17} from '../../../types/models' 16} from '../../../types/models'
@@ -20,10 +19,10 @@ import { getUpdateActivityPubUrl } from '../url'
20import { getActorsInvolvedInVideo } from './shared' 19import { getActorsInvolvedInVideo } from './shared'
21import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils' 20import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils'
22 21
23async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: Transaction, overriddenByActor?: MActor) { 22async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) {
24 const video = videoArg as MVideoAP 23 if (!videoArg.hasPrivacyForFederation()) return undefined
25 24
26 if (!video.hasPrivacyForFederation()) return undefined 25 const video = await videoArg.lightAPToFullAP(transaction)
27 26
28 logger.info('Creating job to update video %s.', video.url) 27 logger.info('Creating job to update video %s.', video.url)
29 28
@@ -31,11 +30,6 @@ async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: T
31 30
32 const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) 31 const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString())
33 32
34 // Needed to build the AP object
35 if (!video.VideoCaptions) {
36 video.VideoCaptions = await video.$get('VideoCaptions', { transaction })
37 }
38
39 const videoObject = await video.toActivityPubObject() 33 const videoObject = await video.toActivityPubObject()
40 const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) 34 const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
41 35
diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts
index bd0c54b0c..d7e251153 100644
--- a/server/lib/activitypub/videos/federate.ts
+++ b/server/lib/activitypub/videos/federate.ts
@@ -1,10 +1,9 @@
1import { Transaction } from 'sequelize/types' 1import { Transaction } from 'sequelize/types'
2import { isArray } from '@server/helpers/custom-validators/misc' 2import { MVideoAP, MVideoAPLight } from '@server/types/models'
3import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models'
4import { sendCreateVideo, sendUpdateVideo } from '../send' 3import { sendCreateVideo, sendUpdateVideo } from '../send'
5import { shareVideoByServerAndChannel } from '../share' 4import { shareVideoByServerAndChannel } from '../share'
6 5
7async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) { 6async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) {
8 const video = videoArg as MVideoAP 7 const video = videoArg as MVideoAP
9 8
10 if ( 9 if (
@@ -13,13 +12,7 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid
13 // Check the video is public/unlisted and published 12 // Check the video is public/unlisted and published
14 video.hasPrivacyForFederation() && video.hasStateForFederation() 13 video.hasPrivacyForFederation() && video.hasStateForFederation()
15 ) { 14 ) {
16 // Fetch more attributes that we will need to serialize in AP object 15 const video = await videoArg.lightAPToFullAP(transaction)
17 if (isArray(video.VideoCaptions) === false) {
18 video.VideoCaptions = await video.$get('VideoCaptions', {
19 attributes: [ 'filename', 'language' ],
20 transaction
21 })
22 }
23 16
24 if (isNewVideo) { 17 if (isNewVideo) {
25 // Now we'll add the video's meta data to our followers 18 // Now we'll add the video's meta data to our followers
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
index c0b92c93d..7c5c73139 100644
--- a/server/lib/activitypub/videos/shared/abstract-builder.ts
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -3,6 +3,7 @@ import { deleteAllModels, filterNonExistingModels } from '@server/helpers/databa
3import { logger, LoggerTagsFn } from '@server/helpers/logger' 3import { logger, LoggerTagsFn } from '@server/helpers/logger'
4import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' 4import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
5import { setVideoTags } from '@server/lib/video' 5import { setVideoTags } from '@server/lib/video'
6import { StoryboardModel } from '@server/models/video/storyboard'
6import { VideoCaptionModel } from '@server/models/video/video-caption' 7import { VideoCaptionModel } from '@server/models/video/video-caption'
7import { VideoFileModel } from '@server/models/video/video-file' 8import { VideoFileModel } from '@server/models/video/video-file'
8import { VideoLiveModel } from '@server/models/video/video-live' 9import { VideoLiveModel } from '@server/models/video/video-live'
@@ -24,6 +25,7 @@ import {
24 getFileAttributesFromUrl, 25 getFileAttributesFromUrl,
25 getLiveAttributesFromObject, 26 getLiveAttributesFromObject,
26 getPreviewFromIcons, 27 getPreviewFromIcons,
28 getStoryboardAttributeFromObject,
27 getStreamingPlaylistAttributesFromObject, 29 getStreamingPlaylistAttributesFromObject,
28 getTagsFromObject, 30 getTagsFromObject,
29 getThumbnailFromIcons 31 getThumbnailFromIcons
@@ -107,6 +109,16 @@ export abstract class APVideoAbstractBuilder {
107 } 109 }
108 } 110 }
109 111
112 protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) {
113 const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t)
114 if (existingStoryboard) await existingStoryboard.destroy({ transaction: t })
115
116 const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject)
117 if (!storyboardAttributes) return
118
119 return StoryboardModel.create(storyboardAttributes, { transaction: t })
120 }
121
110 protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { 122 protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) {
111 const attributes = getLiveAttributesFromObject(video, this.videoObject) 123 const attributes = getLiveAttributesFromObject(video, this.videoObject)
112 const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) 124 const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true })
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts
index 77321d8a5..e6d7bc23c 100644
--- a/server/lib/activitypub/videos/shared/creator.ts
+++ b/server/lib/activitypub/videos/shared/creator.ts
@@ -48,6 +48,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
48 await this.setTrackers(videoCreated, t) 48 await this.setTrackers(videoCreated, t)
49 await this.insertOrReplaceCaptions(videoCreated, t) 49 await this.insertOrReplaceCaptions(videoCreated, t)
50 await this.insertOrReplaceLive(videoCreated, t) 50 await this.insertOrReplaceLive(videoCreated, t)
51 await this.insertOrReplaceStoryboard(videoCreated, t)
51 52
52 // We added a video in this channel, set it as updated 53 // We added a video in this channel, set it as updated
53 await channel.setAsUpdated(t) 54 await channel.setAsUpdated(t)
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index 8fd0a816c..a9e0bed97 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -1,6 +1,6 @@
1import { maxBy, minBy } from 'lodash' 1import { maxBy, minBy } from 'lodash'
2import { decode as magnetUriDecode } from 'magnet-uri' 2import { decode as magnetUriDecode } from 'magnet-uri'
3import { basename } from 'path' 3import { basename, extname } from 'path'
4import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos' 4import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos'
5import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos' 5import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos'
6import { logger } from '@server/helpers/logger' 6import { logger } from '@server/helpers/logger'
@@ -25,6 +25,9 @@ import {
25 VideoStreamingPlaylistType 25 VideoStreamingPlaylistType
26} from '@shared/models' 26} from '@shared/models'
27import { getDurationFromActivityStream } from '../../activity' 27import { getDurationFromActivityStream } from '../../activity'
28import { isArray } from '@server/helpers/custom-validators/misc'
29import { generateImageFilename } from '@server/helpers/image-utils'
30import { arrayify } from '@shared/core-utils'
28 31
29function getThumbnailFromIcons (videoObject: VideoObject) { 32function getThumbnailFromIcons (videoObject: VideoObject) {
30 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) 33 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@@ -166,6 +169,26 @@ function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObje
166 })) 169 }))
167} 170}
168 171
172function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) {
173 if (!isArray(videoObject.preview)) return undefined
174
175 const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard'))
176 if (!storyboard) return undefined
177
178 const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg')
179
180 return {
181 filename: generateImageFilename(extname(url.href)),
182 totalHeight: url.height,
183 totalWidth: url.width,
184 spriteHeight: url.tileHeight,
185 spriteWidth: url.tileWidth,
186 spriteDuration: getDurationFromActivityStream(url.tileDuration),
187 fileUrl: url.href,
188 videoId: video.id
189 }
190}
191
169function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { 192function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
170 const privacy = to.includes(ACTIVITY_PUB.PUBLIC) 193 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
171 ? VideoPrivacy.PUBLIC 194 ? VideoPrivacy.PUBLIC
@@ -228,6 +251,7 @@ export {
228 251
229 getLiveAttributesFromObject, 252 getLiveAttributesFromObject,
230 getCaptionAttributesFromObject, 253 getCaptionAttributesFromObject,
254 getStoryboardAttributeFromObject,
231 255
232 getVideoAttributesFromObject 256 getVideoAttributesFromObject
233} 257}
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts
index 6ddd2301b..3a0886523 100644
--- a/server/lib/activitypub/videos/updater.ts
+++ b/server/lib/activitypub/videos/updater.ts
@@ -57,6 +57,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
57 await Promise.all([ 57 await Promise.all([
58 runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), 58 runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
59 runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), 59 runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
60 runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)),
60 this.setOrDeleteLive(videoUpdated), 61 this.setOrDeleteLive(videoUpdated),
61 this.setPreview(videoUpdated) 62 this.setPreview(videoUpdated)
62 ]) 63 ])
@@ -138,6 +139,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
138 await this.insertOrReplaceCaptions(videoUpdated, t) 139 await this.insertOrReplaceCaptions(videoUpdated, t)
139 } 140 }
140 141
142 private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) {
143 await this.insertOrReplaceStoryboard(videoUpdated, t)
144 }
145
141 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { 146 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
142 if (!this.video.isLive) return 147 if (!this.video.isLive) return
143 148
diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts
index e5853f7d6..59cec7215 100644
--- a/server/lib/files-cache/index.ts
+++ b/server/lib/files-cache/index.ts
@@ -1,3 +1,4 @@
1export * from './videos-preview-cache'
2export * from './videos-caption-cache' 1export * from './videos-caption-cache'
2export * from './videos-preview-cache'
3export * from './videos-storyboard-cache'
3export * from './videos-torrent-cache' 4export * from './videos-torrent-cache'
diff --git a/server/lib/files-cache/videos-storyboard-cache.ts b/server/lib/files-cache/videos-storyboard-cache.ts
new file mode 100644
index 000000000..b0a55104f
--- /dev/null
+++ b/server/lib/files-cache/videos-storyboard-cache.ts
@@ -0,0 +1,53 @@
1import { join } from 'path'
2import { logger } from '@server/helpers/logger'
3import { doRequestAndSaveToFile } from '@server/helpers/requests'
4import { StoryboardModel } from '@server/models/video/storyboard'
5import { FILES_CACHE } from '../../initializers/constants'
6import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
7
8class VideosStoryboardCache extends AbstractVideoStaticFileCache <string> {
9
10 private static instance: VideosStoryboardCache
11
12 private constructor () {
13 super()
14 }
15
16 static get Instance () {
17 return this.instance || (this.instance = new this())
18 }
19
20 async getFilePathImpl (filename: string) {
21 const storyboard = await StoryboardModel.loadWithVideoByFilename(filename)
22 if (!storyboard) return undefined
23
24 if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() }
25
26 return this.loadRemoteFile(storyboard.filename)
27 }
28
29 // Key is the storyboard filename
30 protected async loadRemoteFile (key: string) {
31 const storyboard = await StoryboardModel.loadWithVideoByFilename(key)
32 if (!storyboard) return undefined
33
34 const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename)
35 const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video)
36
37 try {
38 await doRequestAndSaveToFile(remoteUrl, destPath)
39
40 logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath)
41
42 return { isOwned: false, path: destPath }
43 } catch (err) {
44 logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err })
45
46 return undefined
47 }
48 }
49}
50
51export {
52 VideosStoryboardCache
53}
diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts
new file mode 100644
index 000000000..652cac272
--- /dev/null
+++ b/server/lib/job-queue/handlers/generate-storyboard.ts
@@ -0,0 +1,138 @@
1import { Job } from 'bullmq'
2import { join } from 'path'
3import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
4import { generateImageFilename, getImageSize } from '@server/helpers/image-utils'
5import { logger, loggerTagsFactory } from '@server/helpers/logger'
6import { CONFIG } from '@server/initializers/config'
7import { STORYBOARD } from '@server/initializers/constants'
8import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
9import { VideoPathManager } from '@server/lib/video-path-manager'
10import { StoryboardModel } from '@server/models/video/storyboard'
11import { VideoModel } from '@server/models/video/video'
12import { MVideo } from '@server/types/models'
13import { FFmpegImage, isAudioFile } from '@shared/ffmpeg'
14import { GenerateStoryboardPayload } from '@shared/models'
15
16const lTagsBase = loggerTagsFactory('storyboard')
17
18async function processGenerateStoryboard (job: Job): Promise<void> {
19 const payload = job.data as GenerateStoryboardPayload
20 const lTags = lTagsBase(payload.videoUUID)
21
22 logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags)
23
24 const video = await VideoModel.loadFull(payload.videoUUID)
25 if (!video) {
26 logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags)
27 return
28 }
29
30 const inputFile = video.getMaxQualityFile()
31
32 await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
33 const isAudio = await isAudioFile(videoPath)
34
35 if (isAudio) {
36 logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
37 return
38 }
39
40 const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
41
42 const filename = generateImageFilename()
43 const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
44
45 const totalSprites = buildTotalSprites(video)
46 const spriteDuration = Math.round(video.duration / totalSprites)
47
48 const spritesCount = findGridSize({
49 toFind: totalSprites,
50 maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
51 })
52
53 logger.debug(
54 'Generating storyboard from video of %s to %s', video.uuid, destination,
55 { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration }
56 )
57
58 await ffmpeg.generateStoryboardFromVideo({
59 destination,
60 path: videoPath,
61 sprites: {
62 size: STORYBOARD.SPRITE_SIZE,
63 count: spritesCount,
64 duration: spriteDuration
65 }
66 })
67
68 const imageSize = await getImageSize(destination)
69
70 const existing = await StoryboardModel.loadByVideo(video.id)
71 if (existing) await existing.destroy()
72
73 await StoryboardModel.create({
74 filename,
75 totalHeight: imageSize.height,
76 totalWidth: imageSize.width,
77 spriteHeight: STORYBOARD.SPRITE_SIZE.height,
78 spriteWidth: STORYBOARD.SPRITE_SIZE.width,
79 spriteDuration,
80 videoId: video.id
81 })
82
83 logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags)
84 })
85
86 if (payload.federate) {
87 await federateVideoIfNeeded(video, false)
88 }
89}
90
91// ---------------------------------------------------------------------------
92
93export {
94 processGenerateStoryboard
95}
96
97function buildTotalSprites (video: MVideo) {
98 const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width
99 const totalSprites = Math.min(Math.ceil(video.duration), maxSprites)
100
101 // We can generate a single line
102 if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites
103
104 return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT)
105}
106
107function findGridSize (options: {
108 toFind: number
109 maxEdgeCount: number
110}) {
111 const { toFind, maxEdgeCount } = options
112
113 for (let i = 1; i <= maxEdgeCount; i++) {
114 for (let j = i; j <= maxEdgeCount; j++) {
115 if (toFind === i * j) return { width: j, height: i }
116 }
117 }
118
119 throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
120}
121
122function findGridFit (value: number, maxMultiplier: number) {
123 for (let i = value; i--; i > 0) {
124 if (!isPrimeWithin(i, maxMultiplier)) return i
125 }
126
127 throw new Error('Could not find prime number below ' + value)
128}
129
130function isPrimeWithin (value: number, maxMultiplier: number) {
131 if (value < 2) return false
132
133 for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) {
134 if (value % i === 0 && value / i <= maxMultiplier) return false
135 }
136
137 return true
138}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index cdd362f6e..c1355dcef 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -306,6 +306,15 @@ async function afterImportSuccess (options: {
306 Notifier.Instance.notifyOnNewVideoIfNeeded(video) 306 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
307 } 307 }
308 308
309 // Generate the storyboard in the job queue, and don't forget to federate an update after
310 await JobQueue.Instance.createJob({
311 type: 'generate-video-storyboard' as 'generate-video-storyboard',
312 payload: {
313 videoUUID: video.uuid,
314 federate: true
315 }
316 })
317
309 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { 318 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
310 await JobQueue.Instance.createJob( 319 await JobQueue.Instance.createJob(
311 await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) 320 await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 49feb53f2..95d4f5e64 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -1,6 +1,8 @@
1import { Job } from 'bullmq' 1import { Job } from 'bullmq'
2import { readdir, remove } from 'fs-extra' 2import { readdir, remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { peertubeTruncate } from '@server/helpers/core-utils'
5import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
4import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 6import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
5import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' 7import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
6import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' 8import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
@@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv
20import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' 22import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'
21import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 23import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
22import { logger, loggerTagsFactory } from '../../../helpers/logger' 24import { logger, loggerTagsFactory } from '../../../helpers/logger'
23import { peertubeTruncate } from '@server/helpers/core-utils' 25import { JobQueue } from '../job-queue'
24import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
25 26
26const lTags = loggerTagsFactory('live', 'job') 27const lTags = loggerTagsFactory('live', 'job')
27 28
@@ -147,6 +148,8 @@ async function saveReplayToExternalVideo (options: {
147 } 148 }
148 149
149 await moveToNextState({ video: replayVideo, isNewVideo: true }) 150 await moveToNextState({ video: replayVideo, isNewVideo: true })
151
152 await createStoryboardJob(replayVideo)
150} 153}
151 154
152async function replaceLiveByReplay (options: { 155async function replaceLiveByReplay (options: {
@@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: {
186 189
187 await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) 190 await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
188 191
192 // FIXME: should not happen in this function
189 if (permanentLive) { // Remove session replay 193 if (permanentLive) { // Remove session replay
190 await remove(replayDirectory) 194 await remove(replayDirectory)
191 } else { // We won't stream again in this live, we can delete the base replay directory 195 } else { // We won't stream again in this live, we can delete the base replay directory
@@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: {
213 217
214 // We consider this is a new video 218 // We consider this is a new video
215 await moveToNextState({ video: videoWithFiles, isNewVideo: true }) 219 await moveToNextState({ video: videoWithFiles, isNewVideo: true })
220
221 await createStoryboardJob(videoWithFiles)
216} 222}
217 223
218async function assignReplayFilesToVideo (options: { 224async function assignReplayFilesToVideo (options: {
@@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: {
277 logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) 283 logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
278 } 284 }
279} 285}
286
287function createStoryboardJob (video: MVideo) {
288 return JobQueue.Instance.createJob({
289 type: 'generate-video-storyboard' as 'generate-video-storyboard',
290 payload: {
291 videoUUID: video.uuid,
292 federate: true
293 }
294 })
295}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 03f6fbea7..177bca285 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -25,6 +25,7 @@ import {
25 DeleteResumableUploadMetaFilePayload, 25 DeleteResumableUploadMetaFilePayload,
26 EmailPayload, 26 EmailPayload,
27 FederateVideoPayload, 27 FederateVideoPayload,
28 GenerateStoryboardPayload,
28 JobState, 29 JobState,
29 JobType, 30 JobType,
30 ManageVideoTorrentPayload, 31 ManageVideoTorrentPayload,
@@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending'
65import { processVideoStudioEdition } from './handlers/video-studio-edition' 66import { processVideoStudioEdition } from './handlers/video-studio-edition'
66import { processVideoTranscoding } from './handlers/video-transcoding' 67import { processVideoTranscoding } from './handlers/video-transcoding'
67import { processVideosViewsStats } from './handlers/video-views-stats' 68import { processVideosViewsStats } from './handlers/video-views-stats'
69import { processGenerateStoryboard } from './handlers/generate-storyboard'
68 70
69export type CreateJobArgument = 71export type CreateJobArgument =
70 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 72 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -91,7 +93,8 @@ export type CreateJobArgument =
91 { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | 93 { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
92 { type: 'notify', payload: NotifyPayload } | 94 { type: 'notify', payload: NotifyPayload } |
93 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | 95 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
94 { type: 'federate-video', payload: FederateVideoPayload } 96 { type: 'federate-video', payload: FederateVideoPayload } |
97 { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
95 98
96export type CreateJobOptions = { 99export type CreateJobOptions = {
97 delay?: number 100 delay?: number
@@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
122 'video-redundancy': processVideoRedundancy, 125 'video-redundancy': processVideoRedundancy,
123 'video-studio-edition': processVideoStudioEdition, 126 'video-studio-edition': processVideoStudioEdition,
124 'video-transcoding': processVideoTranscoding, 127 'video-transcoding': processVideoTranscoding,
125 'videos-views-stats': processVideosViewsStats 128 'videos-views-stats': processVideosViewsStats,
129 'generate-video-storyboard': processGenerateStoryboard
126} 130}
127 131
128const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { 132const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
@@ -141,10 +145,11 @@ const jobTypes: JobType[] = [
141 'after-video-channel-import', 145 'after-video-channel-import',
142 'email', 146 'email',
143 'federate-video', 147 'federate-video',
144 'transcoding-job-builder', 148 'generate-video-storyboard',
145 'manage-video-torrent', 149 'manage-video-torrent',
146 'move-to-object-storage', 150 'move-to-object-storage',
147 'notify', 151 'notify',
152 'transcoding-job-builder',
148 'video-channel-import', 153 'video-channel-import',
149 'video-file-import', 154 'video-file-import',
150 'video-import', 155 'video-import',
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 8430b2227..48d9986b5 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -325,8 +325,8 @@ class Redis {
325 const value = await this.getValue('resumable-upload-' + uploadId) 325 const value = await this.getValue('resumable-upload-' + uploadId)
326 326
327 return value 327 return value
328 ? JSON.parse(value) 328 ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } }
329 : '' 329 : undefined
330 } 330 }
331 331
332 deleteUploadSession (uploadId: string) { 332 deleteUploadSession (uploadId: string) {
diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts
index 7cc8f20bc..a499db422 100644
--- a/server/lib/transcoding/web-transcoding.ts
+++ b/server/lib/transcoding/web-transcoding.ts
@@ -9,6 +9,7 @@ import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVOD
9import { VideoResolution, VideoStorage } from '@shared/models' 9import { VideoResolution, VideoStorage } from '@shared/models'
10import { CONFIG } from '../../initializers/config' 10import { CONFIG } from '../../initializers/config'
11import { VideoFileModel } from '../../models/video/video-file' 11import { VideoFileModel } from '../../models/video/video-file'
12import { JobQueue } from '../job-queue'
12import { generateWebTorrentVideoFilename } from '../paths' 13import { generateWebTorrentVideoFilename } from '../paths'
13import { buildFileMetadata } from '../video-file' 14import { buildFileMetadata } from '../video-file'
14import { VideoPathManager } from '../video-path-manager' 15import { VideoPathManager } from '../video-path-manager'
@@ -198,7 +199,8 @@ export async function mergeAudioVideofile (options: {
198 return onWebTorrentVideoFileTranscoding({ 199 return onWebTorrentVideoFileTranscoding({
199 video, 200 video,
200 videoFile: inputVideoFile, 201 videoFile: inputVideoFile,
201 videoOutputPath 202 videoOutputPath,
203 wasAudioFile: true
202 }) 204 })
203 }) 205 })
204 206
@@ -212,8 +214,9 @@ export async function onWebTorrentVideoFileTranscoding (options: {
212 video: MVideoFullLight 214 video: MVideoFullLight
213 videoFile: MVideoFile 215 videoFile: MVideoFile
214 videoOutputPath: string 216 videoOutputPath: string
217 wasAudioFile?: boolean // default false
215}) { 218}) {
216 const { video, videoFile, videoOutputPath } = options 219 const { video, videoFile, videoOutputPath, wasAudioFile } = options
217 220
218 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) 221 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
219 222
@@ -242,6 +245,17 @@ export async function onWebTorrentVideoFileTranscoding (options: {
242 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 245 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
243 video.VideoFiles = await video.$get('VideoFiles') 246 video.VideoFiles = await video.$get('VideoFiles')
244 247
248 if (wasAudioFile) {
249 await JobQueue.Instance.createJob({
250 type: 'generate-video-storyboard' as 'generate-video-storyboard',
251 payload: {
252 videoUUID: video.uuid,
253 // No need to federate, we process these jobs sequentially
254 federate: false
255 }
256 })
257 }
258
245 return { video, videoFile } 259 return { video, videoFile }
246 } finally { 260 } finally {
247 mutexReleaser() 261 mutexReleaser()
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index a0074cb24..7029a857f 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -25,6 +25,7 @@ const customConfigUpdateValidator = [
25 body('cache.previews.size').isInt(), 25 body('cache.previews.size').isInt(),
26 body('cache.captions.size').isInt(), 26 body('cache.captions.size').isInt(),
27 body('cache.torrents.size').isInt(), 27 body('cache.torrents.size').isInt(),
28 body('cache.storyboards.size').isInt(),
28 29
29 body('signup.enabled').isBoolean(), 30 body('signup.enabled').isBoolean(),
30 body('signup.limit').isInt(), 31 body('signup.limit').isInt(),
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index f2001e432..4179545b8 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -5,6 +5,7 @@ import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
5import { VideoViewsManager } from '@server/lib/views/video-views-manager' 5import { VideoViewsManager } from '@server/lib/views/video-views-manager'
6import { uuidToShort } from '@shared/extra-utils' 6import { uuidToShort } from '@shared/extra-utils'
7import { 7import {
8 ActivityPubStoryboard,
8 ActivityTagObject, 9 ActivityTagObject,
9 ActivityUrlObject, 10 ActivityUrlObject,
10 Video, 11 Video,
@@ -347,29 +348,17 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
347 name: t.name 348 name: t.name
348 })) 349 }))
349 350
350 let language 351 const language = video.language
351 if (video.language) { 352 ? { identifier: video.language, name: getLanguageLabel(video.language) }
352 language = { 353 : undefined
353 identifier: video.language,
354 name: getLanguageLabel(video.language)
355 }
356 }
357 354
358 let category 355 const category = video.category
359 if (video.category) { 356 ? { identifier: video.category + '', name: getCategoryLabel(video.category) }
360 category = { 357 : undefined
361 identifier: video.category + '',
362 name: getCategoryLabel(video.category)
363 }
364 }
365 358
366 let licence 359 const licence = video.licence
367 if (video.licence) { 360 ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) }
368 licence = { 361 : undefined
369 identifier: video.licence + '',
370 name: getLicenceLabel(video.licence)
371 }
372 }
373 362
374 const url: ActivityUrlObject[] = [ 363 const url: ActivityUrlObject[] = [
375 // HTML url should be the first element in the array so Mastodon correctly displays the embed 364 // HTML url should be the first element in the array so Mastodon correctly displays the embed
@@ -465,6 +454,8 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
465 height: i.height 454 height: i.height
466 })), 455 })),
467 456
457 preview: buildPreviewAPAttribute(video),
458
468 url, 459 url,
469 460
470 likes: getLocalVideoLikesActivityPubUrl(video), 461 likes: getLocalVideoLikesActivityPubUrl(video),
@@ -541,3 +532,30 @@ function buildLiveAPAttributes (video: MVideoAP) {
541 latencyMode: video.VideoLive.latencyMode 532 latencyMode: video.VideoLive.latencyMode
542 } 533 }
543} 534}
535
536function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {
537 if (!video.Storyboard) return undefined
538
539 const storyboard = video.Storyboard
540
541 return [
542 {
543 type: 'Image',
544 rel: [ 'storyboard' ],
545 url: [
546 {
547 mediaType: 'image/jpeg',
548
549 href: storyboard.getOriginFileUrl(video),
550
551 width: storyboard.totalWidth,
552 height: storyboard.totalHeight,
553
554 tileWidth: storyboard.spriteWidth,
555 tileHeight: storyboard.spriteHeight,
556 tileDuration: getActivityStreamDuration(storyboard.spriteDuration)
557 }
558 ]
559 }
560 ]
561}
diff --git a/server/models/video/storyboard.ts b/server/models/video/storyboard.ts
new file mode 100644
index 000000000..65a044c98
--- /dev/null
+++ b/server/models/video/storyboard.ts
@@ -0,0 +1,169 @@
1import { remove } from 'fs-extra'
2import { join } from 'path'
3import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
4import { CONFIG } from '@server/initializers/config'
5import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models'
6import { Storyboard } from '@shared/models'
7import { AttributesOnly } from '@shared/typescript-utils'
8import { logger } from '../../helpers/logger'
9import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
10import { VideoModel } from './video'
11import { Transaction } from 'sequelize'
12
13@Table({
14 tableName: 'storyboard',
15 indexes: [
16 {
17 fields: [ 'videoId' ],
18 unique: true
19 },
20 {
21 fields: [ 'filename' ],
22 unique: true
23 }
24 ]
25})
26export class StoryboardModel extends Model<Partial<AttributesOnly<StoryboardModel>>> {
27
28 @AllowNull(false)
29 @Column
30 filename: string
31
32 @AllowNull(false)
33 @Column
34 totalHeight: number
35
36 @AllowNull(false)
37 @Column
38 totalWidth: number
39
40 @AllowNull(false)
41 @Column
42 spriteHeight: number
43
44 @AllowNull(false)
45 @Column
46 spriteWidth: number
47
48 @AllowNull(false)
49 @Column
50 spriteDuration: number
51
52 @AllowNull(true)
53 @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max))
54 fileUrl: string
55
56 @ForeignKey(() => VideoModel)
57 @Column
58 videoId: number
59
60 @BelongsTo(() => VideoModel, {
61 foreignKey: {
62 allowNull: true
63 },
64 onDelete: 'CASCADE'
65 })
66 Video: VideoModel
67
68 @CreatedAt
69 createdAt: Date
70
71 @UpdatedAt
72 updatedAt: Date
73
74 @AfterDestroy
75 static removeInstanceFile (instance: StoryboardModel) {
76 logger.info('Removing storyboard file %s.', instance.filename)
77
78 // Don't block the transaction
79 instance.removeFile()
80 .catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err }))
81 }
82
83 static loadByVideo (videoId: number, transaction?: Transaction): Promise<MStoryboard> {
84 const query = {
85 where: {
86 videoId
87 },
88 transaction
89 }
90
91 return StoryboardModel.findOne(query)
92 }
93
94 static loadByFilename (filename: string): Promise<MStoryboard> {
95 const query = {
96 where: {
97 filename
98 }
99 }
100
101 return StoryboardModel.findOne(query)
102 }
103
104 static loadWithVideoByFilename (filename: string): Promise<MStoryboardVideo> {
105 const query = {
106 where: {
107 filename
108 },
109 include: [
110 {
111 model: VideoModel.unscoped(),
112 required: true
113 }
114 ]
115 }
116
117 return StoryboardModel.findOne(query)
118 }
119
120 // ---------------------------------------------------------------------------
121
122 static async listStoryboardsOf (video: MVideo): Promise<MStoryboardVideo[]> {
123 const query = {
124 where: {
125 videoId: video.id
126 }
127 }
128
129 const storyboards = await StoryboardModel.findAll<MStoryboard>(query)
130
131 return storyboards.map(s => Object.assign(s, { Video: video }))
132 }
133
134 // ---------------------------------------------------------------------------
135
136 getOriginFileUrl (video: MVideo) {
137 if (video.isOwned()) {
138 return WEBSERVER.URL + this.getLocalStaticPath()
139 }
140
141 return this.fileUrl
142 }
143
144 getLocalStaticPath () {
145 return LAZY_STATIC_PATHS.STORYBOARDS + this.filename
146 }
147
148 getPath () {
149 return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename)
150 }
151
152 removeFile () {
153 return remove(this.getPath())
154 }
155
156 toFormattedJSON (this: MStoryboardVideo): Storyboard {
157 return {
158 storyboardPath: this.getLocalStaticPath(),
159
160 totalHeight: this.totalHeight,
161 totalWidth: this.totalWidth,
162
163 spriteWidth: this.spriteWidth,
164 spriteHeight: this.spriteHeight,
165
166 spriteDuration: this.spriteDuration
167 }
168 }
169}
diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts
index 1fb1cae82..dd4cefd65 100644
--- a/server/models/video/video-caption.ts
+++ b/server/models/video/video-caption.ts
@@ -15,7 +15,7 @@ import {
15 Table, 15 Table,
16 UpdatedAt 16 UpdatedAt
17} from 'sequelize-typescript' 17} from 'sequelize-typescript'
18import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models' 18import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, MVideoCaptionVideo } from '@server/types/models'
19import { buildUUID } from '@shared/extra-utils' 19import { buildUUID } from '@shared/extra-utils'
20import { AttributesOnly } from '@shared/typescript-utils' 20import { AttributesOnly } from '@shared/typescript-utils'
21import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' 21import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
@@ -225,7 +225,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
225 } 225 }
226 } 226 }
227 227
228 getCaptionStaticPath (this: MVideoCaption) { 228 getCaptionStaticPath (this: MVideoCaptionLanguageUrl) {
229 return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename) 229 return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename)
230 } 230 }
231 231
@@ -233,9 +233,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
233 return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename) 233 return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename)
234 } 234 }
235 235
236 getFileUrl (video: MVideo) { 236 getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideo) {
237 if (!this.Video) this.Video = video as VideoModel
238
239 if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath() 237 if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath()
240 238
241 return this.fileUrl 239 return this.fileUrl
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index f90f2b7f6..0e9a84426 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -58,7 +58,7 @@ import {
58import { AttributesOnly } from '@shared/typescript-utils' 58import { AttributesOnly } from '@shared/typescript-utils'
59import { peertubeTruncate } from '../../helpers/core-utils' 59import { peertubeTruncate } from '../../helpers/core-utils'
60import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 60import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
61import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc' 61import { exists, isArray, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
62import { 62import {
63 isVideoDescriptionValid, 63 isVideoDescriptionValid,
64 isVideoDurationValid, 64 isVideoDurationValid,
@@ -75,6 +75,7 @@ import {
75 MChannel, 75 MChannel,
76 MChannelAccountDefault, 76 MChannelAccountDefault,
77 MChannelId, 77 MChannelId,
78 MStoryboard,
78 MStreamingPlaylist, 79 MStreamingPlaylist,
79 MStreamingPlaylistFilesVideo, 80 MStreamingPlaylistFilesVideo,
80 MUserAccountId, 81 MUserAccountId,
@@ -83,6 +84,8 @@ import {
83 MVideoAccountLight, 84 MVideoAccountLight,
84 MVideoAccountLightBlacklistAllFiles, 85 MVideoAccountLightBlacklistAllFiles,
85 MVideoAP, 86 MVideoAP,
87 MVideoAPLight,
88 MVideoCaptionLanguageUrl,
86 MVideoDetails, 89 MVideoDetails,
87 MVideoFileVideo, 90 MVideoFileVideo,
88 MVideoFormattable, 91 MVideoFormattable,
@@ -126,6 +129,7 @@ import {
126 VideosIdListQueryBuilder, 129 VideosIdListQueryBuilder,
127 VideosModelListQueryBuilder 130 VideosModelListQueryBuilder
128} from './sql/video' 131} from './sql/video'
132import { StoryboardModel } from './storyboard'
129import { TagModel } from './tag' 133import { TagModel } from './tag'
130import { ThumbnailModel } from './thumbnail' 134import { ThumbnailModel } from './thumbnail'
131import { VideoBlacklistModel } from './video-blacklist' 135import { VideoBlacklistModel } from './video-blacklist'
@@ -753,6 +757,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
753 }) 757 })
754 VideoJobInfo: VideoJobInfoModel 758 VideoJobInfo: VideoJobInfoModel
755 759
760 @HasOne(() => StoryboardModel, {
761 foreignKey: {
762 name: 'videoId',
763 allowNull: false
764 },
765 onDelete: 'cascade'
766 })
767 Storyboard: StoryboardModel
768
756 @AfterCreate 769 @AfterCreate
757 static notifyCreate (video: MVideo) { 770 static notifyCreate (video: MVideo) {
758 InternalEventEmitter.Instance.emit('video-created', { video }) 771 InternalEventEmitter.Instance.emit('video-created', { video })
@@ -904,6 +917,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
904 required: false 917 required: false
905 }, 918 },
906 { 919 {
920 model: StoryboardModel.unscoped(),
921 required: false
922 },
923 {
907 attributes: [ 'id', 'url' ], 924 attributes: [ 'id', 'url' ],
908 model: VideoShareModel.unscoped(), 925 model: VideoShareModel.unscoped(),
909 required: false, 926 required: false,
@@ -1768,6 +1785,32 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1768 ) 1785 )
1769 } 1786 }
1770 1787
1788 async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise<MVideoAP> {
1789 const videoAP = this as MVideoAP
1790
1791 const getCaptions = () => {
1792 if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions
1793
1794 return this.$get('VideoCaptions', {
1795 attributes: [ 'filename', 'language', 'fileUrl' ],
1796 transaction
1797 }) as Promise<MVideoCaptionLanguageUrl[]>
1798 }
1799
1800 const getStoryboard = () => {
1801 if (videoAP.Storyboard) return videoAP.Storyboard
1802
1803 return this.$get('Storyboard', { transaction }) as Promise<MStoryboard>
1804 }
1805
1806 const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ])
1807
1808 return Object.assign(this, {
1809 VideoCaptions: captions,
1810 Storyboard: storyboard
1811 })
1812 }
1813
1771 getTruncatedDescription () { 1814 getTruncatedDescription () {
1772 if (!this.description) return null 1815 if (!this.description) return null
1773 1816
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 472cad182..3c752cc3e 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -74,6 +74,9 @@ describe('Test config API validators', function () {
74 }, 74 },
75 torrents: { 75 torrents: {
76 size: 4 76 size: 4
77 },
78 storyboards: {
79 size: 5
77 } 80 }
78 }, 81 },
79 signup: { 82 signup: {
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 400d312d3..c2a7ccd78 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -34,6 +34,7 @@ import './video-comments'
34import './video-files' 34import './video-files'
35import './video-imports' 35import './video-imports'
36import './video-playlists' 36import './video-playlists'
37import './video-storyboards'
37import './video-source' 38import './video-source'
38import './video-studio' 39import './video-studio'
39import './video-token' 40import './video-token'
diff --git a/server/tests/api/check-params/video-storyboards.ts b/server/tests/api/check-params/video-storyboards.ts
new file mode 100644
index 000000000..a43d8fc48
--- /dev/null
+++ b/server/tests/api/check-params/video-storyboards.ts
@@ -0,0 +1,45 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { HttpStatusCode, VideoPrivacy } from '@shared/models'
4import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
5
6describe('Test video storyboards API validator', function () {
7 let server: PeerTubeServer
8
9 let publicVideo: { uuid: string }
10 let privateVideo: { uuid: string }
11
12 // ---------------------------------------------------------------
13
14 before(async function () {
15 this.timeout(30000)
16
17 server = await createSingleServer(1)
18 await setAccessTokensToServers([ server ])
19
20 publicVideo = await server.videos.quickUpload({ name: 'public' })
21 privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE })
22 })
23
24 it('Should fail without a valid uuid', async function () {
25 await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
26 })
27
28 it('Should receive 404 when passing a non existing video id', async function () {
29 await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
30 })
31
32 it('Should not get the private storyboard without the appropriate token', async function () {
33 await server.storyboard.list({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null })
34 await server.storyboard.list({ id: publicVideo.uuid, expectedStatus: HttpStatusCode.OK_200, token: null })
35 })
36
37 it('Should succeed with the correct parameters', async function () {
38 await server.storyboard.list({ id: privateVideo.uuid })
39 await server.storyboard.list({ id: publicVideo.uuid })
40 })
41
42 after(async function () {
43 await cleanupTests([ server ])
44 })
45})
diff --git a/server/tests/api/check-params/videos-overviews.ts b/server/tests/api/check-params/videos-overviews.ts
index f9cdb7ab3..ae7de24dd 100644
--- a/server/tests/api/check-params/videos-overviews.ts
+++ b/server/tests/api/check-params/videos-overviews.ts
@@ -2,7 +2,7 @@
2 2
3import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands' 3import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
4 4
5describe('Test videos overview', function () { 5describe('Test videos overview API validator', function () {
6 let server: PeerTubeServer 6 let server: PeerTubeServer
7 7
8 // --------------------------------------------------------------- 8 // ---------------------------------------------------------------
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index 011ba268c..efa7b50e3 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -46,6 +46,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) {
46 expect(data.cache.previews.size).to.equal(1) 46 expect(data.cache.previews.size).to.equal(1)
47 expect(data.cache.captions.size).to.equal(1) 47 expect(data.cache.captions.size).to.equal(1)
48 expect(data.cache.torrents.size).to.equal(1) 48 expect(data.cache.torrents.size).to.equal(1)
49 expect(data.cache.storyboards.size).to.equal(1)
49 50
50 expect(data.signup.enabled).to.be.true 51 expect(data.signup.enabled).to.be.true
51 expect(data.signup.limit).to.equal(4) 52 expect(data.signup.limit).to.equal(4)
@@ -154,6 +155,7 @@ function checkUpdatedConfig (data: CustomConfig) {
154 expect(data.cache.previews.size).to.equal(2) 155 expect(data.cache.previews.size).to.equal(2)
155 expect(data.cache.captions.size).to.equal(3) 156 expect(data.cache.captions.size).to.equal(3)
156 expect(data.cache.torrents.size).to.equal(4) 157 expect(data.cache.torrents.size).to.equal(4)
158 expect(data.cache.storyboards.size).to.equal(5)
157 159
158 expect(data.signup.enabled).to.be.false 160 expect(data.signup.enabled).to.be.false
159 expect(data.signup.limit).to.equal(5) 161 expect(data.signup.limit).to.equal(5)
@@ -290,6 +292,9 @@ const newCustomConfig: CustomConfig = {
290 }, 292 },
291 torrents: { 293 torrents: {
292 size: 4 294 size: 4
295 },
296 storyboards: {
297 size: 5
293 } 298 }
294 }, 299 },
295 signup: { 300 signup: {
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 357c08199..9c79b3aa6 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -20,3 +20,4 @@ import './videos-history'
20import './videos-overview' 20import './videos-overview'
21import './video-source' 21import './video-source'
22import './video-static-file-privacy' 22import './video-static-file-privacy'
23import './video-storyboard'
diff --git a/server/tests/api/videos/video-storyboard.ts b/server/tests/api/videos/video-storyboard.ts
new file mode 100644
index 000000000..7ccdca8f7
--- /dev/null
+++ b/server/tests/api/videos/video-storyboard.ts
@@ -0,0 +1,184 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { FIXTURE_URLS } from '@server/tests/shared'
5import { areHttpImportTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import {
8 cleanupTests,
9 createMultipleServers,
10 doubleFollow,
11 makeGetRequest,
12 PeerTubeServer,
13 sendRTMPStream,
14 setAccessTokensToServers,
15 setDefaultVideoChannel,
16 stopFfmpeg,
17 waitJobs
18} from '@shared/server-commands'
19
20async function checkStoryboard (options: {
21 server: PeerTubeServer
22 uuid: string
23 tilesCount?: number
24 minSize?: number
25}) {
26 const { server, uuid, tilesCount, minSize = 1000 } = options
27
28 const { storyboards } = await server.storyboard.list({ id: uuid })
29
30 expect(storyboards).to.have.lengthOf(1)
31
32 const storyboard = storyboards[0]
33
34 expect(storyboard.spriteDuration).to.equal(1)
35 expect(storyboard.spriteHeight).to.equal(108)
36 expect(storyboard.spriteWidth).to.equal(192)
37 expect(storyboard.storyboardPath).to.exist
38
39 if (tilesCount) {
40 expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10))
41 expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1))
42 }
43
44 const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
45 expect(body.length).to.be.above(minSize)
46}
47
48describe('Test video storyboard', function () {
49 let servers: PeerTubeServer[]
50
51 before(async function () {
52 this.timeout(120000)
53
54 servers = await createMultipleServers(2)
55 await setAccessTokensToServers(servers)
56 await setDefaultVideoChannel(servers)
57
58 await doubleFollow(servers[0], servers[1])
59 })
60
61 it('Should generate a storyboard after upload without transcoding', async function () {
62 this.timeout(60000)
63
64 // 5s video
65 const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
66 await waitJobs(servers)
67
68 for (const server of servers) {
69 await checkStoryboard({ server, uuid, tilesCount: 5 })
70 }
71 })
72
73 it('Should generate a storyboard after upload without transcoding with a long video', async function () {
74 this.timeout(60000)
75
76 // 124s video
77 const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_very_long_10p.mp4' })
78 await waitJobs(servers)
79
80 for (const server of servers) {
81 await checkStoryboard({ server, uuid, tilesCount: 100 })
82 }
83 })
84
85 it('Should generate a storyboard after upload with transcoding', async function () {
86 this.timeout(60000)
87
88 await servers[0].config.enableMinimumTranscoding()
89
90 // 5s video
91 const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
92 await waitJobs(servers)
93
94 for (const server of servers) {
95 await checkStoryboard({ server, uuid, tilesCount: 5 })
96 }
97 })
98
99 it('Should generate a storyboard after an audio upload', async function () {
100 this.timeout(60000)
101
102 // 6s audio
103 const attributes = { name: 'audio', fixture: 'sample.ogg' }
104 const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' })
105 await waitJobs(servers)
106
107 for (const server of servers) {
108 await checkStoryboard({ server, uuid, tilesCount: 6, minSize: 250 })
109 }
110 })
111
112 it('Should generate a storyboard after HTTP import', async function () {
113 this.timeout(60000)
114
115 if (areHttpImportTestsDisabled()) return
116
117 // 3s video
118 const { video } = await servers[0].imports.importVideo({
119 attributes: {
120 targetUrl: FIXTURE_URLS.goodVideo,
121 channelId: servers[0].store.channel.id,
122 privacy: VideoPrivacy.PUBLIC
123 }
124 })
125 await waitJobs(servers)
126
127 for (const server of servers) {
128 await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 })
129 }
130 })
131
132 it('Should generate a storyboard after torrent import', async function () {
133 this.timeout(60000)
134
135 if (areHttpImportTestsDisabled()) return
136
137 // 10s video
138 const { video } = await servers[0].imports.importVideo({
139 attributes: {
140 magnetUri: FIXTURE_URLS.magnet,
141 channelId: servers[0].store.channel.id,
142 privacy: VideoPrivacy.PUBLIC
143 }
144 })
145 await waitJobs(servers)
146
147 for (const server of servers) {
148 await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 })
149 }
150 })
151
152 it('Should generate a storyboard after a live', async function () {
153 this.timeout(240000)
154
155 await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' })
156
157 const { live, video } = await servers[0].live.quickCreate({
158 saveReplay: true,
159 permanentLive: false,
160 privacy: VideoPrivacy.PUBLIC
161 })
162
163 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
164 await servers[0].live.waitUntilPublished({ videoId: video.id })
165
166 await stopFfmpeg(ffmpegCommand)
167
168 await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id })
169 await waitJobs(servers)
170
171 for (const server of servers) {
172 await checkStoryboard({ server, uuid: video.uuid })
173 }
174 })
175
176 it('Should generate a storyboard with different video durations', async function () {
177 this.timeout(60000)
178
179 })
180
181 after(async function () {
182 await cleanupTests(servers)
183 })
184})
diff --git a/server/tests/fixtures/video_very_long_10p.mp4 b/server/tests/fixtures/video_very_long_10p.mp4
new file mode 100644
index 000000000..852297933
--- /dev/null
+++ b/server/tests/fixtures/video_very_long_10p.mp4
Binary files differ
diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts
index 0ac032290..7f05db666 100644
--- a/server/types/models/video/index.ts
+++ b/server/types/models/video/index.ts
@@ -1,6 +1,7 @@
1export * from './local-video-viewer-watch-section' 1export * from './local-video-viewer-watch-section'
2export * from './local-video-viewer-watch-section' 2export * from './local-video-viewer-watch-section'
3export * from './local-video-viewer' 3export * from './local-video-viewer'
4export * from './storyboard'
4export * from './schedule-video-update' 5export * from './schedule-video-update'
5export * from './tag' 6export * from './tag'
6export * from './thumbnail' 7export * from './thumbnail'
diff --git a/server/types/models/video/storyboard.ts b/server/types/models/video/storyboard.ts
new file mode 100644
index 000000000..a0403d4f0
--- /dev/null
+++ b/server/types/models/video/storyboard.ts
@@ -0,0 +1,15 @@
1import { StoryboardModel } from '@server/models/video/storyboard'
2import { PickWith } from '@shared/typescript-utils'
3import { MVideo } from './video'
4
5type Use<K extends keyof StoryboardModel, M> = PickWith<StoryboardModel, K, M>
6
7// ############################################################################
8
9export type MStoryboard = Omit<StoryboardModel, 'Video'>
10
11// ############################################################################
12
13export type MStoryboardVideo =
14 MStoryboard &
15 Use<'Video', MVideo>
diff --git a/server/types/models/video/video-caption.ts b/server/types/models/video/video-caption.ts
index 8cd801064..d3adec362 100644
--- a/server/types/models/video/video-caption.ts
+++ b/server/types/models/video/video-caption.ts
@@ -11,7 +11,7 @@ export type MVideoCaption = Omit<VideoCaptionModel, 'Video'>
11// ############################################################################ 11// ############################################################################
12 12
13export type MVideoCaptionLanguage = Pick<MVideoCaption, 'language'> 13export type MVideoCaptionLanguage = Pick<MVideoCaption, 'language'>
14export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'getFileUrl'> 14export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'filename' | 'getFileUrl' | 'getCaptionStaticPath'>
15 15
16export type MVideoCaptionVideo = 16export type MVideoCaptionVideo =
17 MVideoCaption & 17 MVideoCaption &
diff --git a/server/types/models/video/video.ts b/server/types/models/video/video.ts
index 8021e56bb..53ee94269 100644
--- a/server/types/models/video/video.ts
+++ b/server/types/models/video/video.ts
@@ -3,6 +3,7 @@ import { VideoModel } from '../../../models/video/video'
3import { MTrackerUrl } from '../server/tracker' 3import { MTrackerUrl } from '../server/tracker'
4import { MUserVideoHistoryTime } from '../user/user-video-history' 4import { MUserVideoHistoryTime } from '../user/user-video-history'
5import { MScheduleVideoUpdate } from './schedule-video-update' 5import { MScheduleVideoUpdate } from './schedule-video-update'
6import { MStoryboard } from './storyboard'
6import { MTag } from './tag' 7import { MTag } from './tag'
7import { MThumbnail } from './thumbnail' 8import { MThumbnail } from './thumbnail'
8import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' 9import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist'
@@ -32,7 +33,7 @@ type Use<K extends keyof VideoModel, M> = PickWith<VideoModel, K, M>
32export type MVideo = 33export type MVideo =
33 Omit<VideoModel, 'VideoChannel' | 'Tags' | 'Thumbnails' | 'VideoPlaylistElements' | 'VideoAbuses' | 34 Omit<VideoModel, 'VideoChannel' | 'Tags' | 'Thumbnails' | 'VideoPlaylistElements' | 'VideoAbuses' |
34 'VideoFiles' | 'VideoStreamingPlaylists' | 'VideoShares' | 'AccountVideoRates' | 'VideoComments' | 'VideoViews' | 'UserVideoHistories' | 35 'VideoFiles' | 'VideoStreamingPlaylists' | 'VideoShares' | 'AccountVideoRates' | 'VideoComments' | 'VideoViews' | 'UserVideoHistories' |
35 'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords'> 36 'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords' | 'Storyboard'>
36 37
37// ############################################################################ 38// ############################################################################
38 39
@@ -173,9 +174,10 @@ export type MVideoAP =
173 Use<'VideoBlacklist', MVideoBlacklistUnfederated> & 174 Use<'VideoBlacklist', MVideoBlacklistUnfederated> &
174 Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & 175 Use<'VideoFiles', MVideoFileRedundanciesOpt[]> &
175 Use<'Thumbnails', MThumbnail[]> & 176 Use<'Thumbnails', MThumbnail[]> &
176 Use<'VideoLive', MVideoLive> 177 Use<'VideoLive', MVideoLive> &
178 Use<'Storyboard', MStoryboard>
177 179
178export type MVideoAPWithoutCaption = Omit<MVideoAP, 'VideoCaptions'> 180export type MVideoAPLight = Omit<MVideoAP, 'VideoCaptions' | 'Storyboard'>
179 181
180export type MVideoDetails = 182export type MVideoDetails =
181 MVideo & 183 MVideo &
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-03 17:55:04 +0000