aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/captions.ts2
-rw-r--r--server/helpers/activitypub.ts13
-rw-r--r--server/helpers/core-utils.ts2
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts26
-rw-r--r--server/initializers/constants.ts8
-rw-r--r--server/initializers/migrations/0480-caption-file-url.ts27
-rw-r--r--server/lib/activitypub/videos.ts68
-rw-r--r--server/lib/files-cache/videos-caption-cache.ts7
-rw-r--r--server/lib/files-cache/videos-preview-cache.ts12
-rw-r--r--server/lib/job-queue/handlers/video-views.ts4
-rw-r--r--server/lib/job-queue/job-queue.ts1
-rw-r--r--server/models/video/thumbnail.ts13
-rw-r--r--server/models/video/video-caption.ts26
-rw-r--r--server/models/video/video-format-utils.ts15
-rw-r--r--server/models/video/video.ts2
-rw-r--r--server/typings/models/video/video-caption.ts1
-rw-r--r--server/typings/models/video/video.ts4
17 files changed, 154 insertions, 77 deletions
diff --git a/server/controllers/api/videos/captions.ts b/server/controllers/api/videos/captions.ts
index 37481d12f..fd7b165fb 100644
--- a/server/controllers/api/videos/captions.ts
+++ b/server/controllers/api/videos/captions.ts
@@ -66,7 +66,7 @@ async function addVideoCaption (req: express.Request, res: express.Response) {
66 await moveAndProcessCaptionFile(videoCaptionPhysicalFile, videoCaption) 66 await moveAndProcessCaptionFile(videoCaptionPhysicalFile, videoCaption)
67 67
68 await sequelizeTypescript.transaction(async t => { 68 await sequelizeTypescript.transaction(async t => {
69 await VideoCaptionModel.insertOrReplaceLanguage(video.id, req.params.captionLanguage, t) 69 await VideoCaptionModel.insertOrReplaceLanguage(video.id, req.params.captionLanguage, null, t)
70 70
71 // Update video update 71 // Update video update
72 await federateVideoIfNeeded(video, false, t) 72 await federateVideoIfNeeded(video, false, t)
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index 239d8291d..9f9e8fba7 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -2,11 +2,11 @@ import * as Bluebird from 'bluebird'
2import validator from 'validator' 2import validator from 'validator'
3import { ResultList } from '../../shared/models' 3import { ResultList } from '../../shared/models'
4import { Activity } from '../../shared/models/activitypub' 4import { Activity } from '../../shared/models/activitypub'
5import { ACTIVITY_PUB } from '../initializers/constants' 5import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants'
6import { signJsonLDObject } from './peertube-crypto' 6import { signJsonLDObject } from './peertube-crypto'
7import { pageToStartAndCount } from './core-utils' 7import { pageToStartAndCount } from './core-utils'
8import { parse } from 'url' 8import { parse } from 'url'
9import { MActor } from '../typings/models' 9import { MActor, MVideoAccountLight } from '../typings/models'
10 10
11function activityPubContextify <T> (data: T) { 11function activityPubContextify <T> (data: T) {
12 return Object.assign(data, { 12 return Object.assign(data, {
@@ -167,6 +167,12 @@ function checkUrlsSameHost (url1: string, url2: string) {
167 return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() 167 return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase()
168} 168}
169 169
170function buildRemoteVideoBaseUrl (video: MVideoAccountLight, path: string) {
171 const host = video.VideoChannel.Account.Actor.Server.host
172
173 return REMOTE_SCHEME.HTTP + '://' + host + path
174}
175
170// --------------------------------------------------------------------------- 176// ---------------------------------------------------------------------------
171 177
172export { 178export {
@@ -174,5 +180,6 @@ export {
174 getAPId, 180 getAPId,
175 activityPubContextify, 181 activityPubContextify,
176 activityPubCollectionPagination, 182 activityPubCollectionPagination,
177 buildSignedActivity 183 buildSignedActivity,
184 buildRemoteVideoBaseUrl
178} 185}
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 7e8252aa4..519dc83d0 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -199,6 +199,8 @@ function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex')
199 return createHash('sha1').update(str).digest(encoding) 199 return createHash('sha1').update(str).digest(encoding)
200} 200}
201 201
202
203
202function execShell (command: string, options?: ExecOptions) { 204function execShell (command: string, options?: ExecOptions) {
203 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { 205 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
204 exec(command, options, (err, stdout, stderr) => { 206 exec(command, options, (err, stdout, stderr) => {
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 224f03f4e..22b5e14a2 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -51,6 +51,10 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
51 logger.debug('Video has invalid captions', { video }) 51 logger.debug('Video has invalid captions', { video })
52 return false 52 return false
53 } 53 }
54 if (!setValidRemoteIcon(video)) {
55 logger.debug('Video has invalid icons', { video })
56 return false
57 }
54 58
55 // Default attributes 59 // Default attributes
56 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED 60 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
@@ -73,7 +77,6 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
73 isDateValid(video.updated) && 77 isDateValid(video.updated) &&
74 (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && 78 (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) &&
75 (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && 79 (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) &&
76 isRemoteVideoIconValid(video.icon) &&
77 video.url.length !== 0 && 80 video.url.length !== 0 &&
78 video.attributedTo.length !== 0 81 video.attributedTo.length !== 0
79} 82}
@@ -132,6 +135,8 @@ function setValidRemoteCaptions (video: any) {
132 if (Array.isArray(video.subtitleLanguage) === false) return false 135 if (Array.isArray(video.subtitleLanguage) === false) return false
133 136
134 video.subtitleLanguage = video.subtitleLanguage.filter(caption => { 137 video.subtitleLanguage = video.subtitleLanguage.filter(caption => {
138 if (!isActivityPubUrlValid(caption.url)) caption.url = null
139
135 return isRemoteStringIdentifierValid(caption) 140 return isRemoteStringIdentifierValid(caption)
136 }) 141 })
137 142
@@ -150,12 +155,19 @@ function isRemoteVideoContentValid (mediaType: string, content: string) {
150 return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content) 155 return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content)
151} 156}
152 157
153function isRemoteVideoIconValid (icon: any) { 158function setValidRemoteIcon (video: any) {
154 return icon.type === 'Image' && 159 if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ]
155 isActivityPubUrlValid(icon.url) && 160 if (!video.icon) video.icon = []
156 icon.mediaType === 'image/jpeg' && 161
157 validator.isInt(icon.width + '', { min: 0 }) && 162 video.icon = video.icon.filter(icon => {
158 validator.isInt(icon.height + '', { min: 0 }) 163 return icon.type === 'Image' &&
164 isActivityPubUrlValid(icon.url) &&
165 icon.mediaType === 'image/jpeg' &&
166 validator.isInt(icon.width + '', { min: 0 }) &&
167 validator.isInt(icon.height + '', { min: 0 })
168 })
169
170 return video.icon.length !== 0
159} 171}
160 172
161function setValidRemoteVideoUrls (video: any) { 173function setValidRemoteVideoUrls (video: any) {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 64803b1db..3a9946bba 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
14 14
15// --------------------------------------------------------------------------- 15// ---------------------------------------------------------------------------
16 16
17const LAST_MIGRATION_VERSION = 475 17const LAST_MIGRATION_VERSION = 480
18 18
19// --------------------------------------------------------------------------- 19// ---------------------------------------------------------------------------
20 20
@@ -541,11 +541,13 @@ let STATIC_MAX_AGE = {
541// Videos thumbnail size 541// Videos thumbnail size
542const THUMBNAILS_SIZE = { 542const THUMBNAILS_SIZE = {
543 width: 223, 543 width: 223,
544 height: 122 544 height: 122,
545 minWidth: 150
545} 546}
546const PREVIEWS_SIZE = { 547const PREVIEWS_SIZE = {
547 width: 850, 548 width: 850,
548 height: 480 549 height: 480,
550 minWidth: 400
549} 551}
550const AVATARS_SIZE = { 552const AVATARS_SIZE = {
551 width: 120, 553 width: 120,
diff --git a/server/initializers/migrations/0480-caption-file-url.ts b/server/initializers/migrations/0480-caption-file-url.ts
new file mode 100644
index 000000000..7d8a3d4b9
--- /dev/null
+++ b/server/initializers/migrations/0480-caption-file-url.ts
@@ -0,0 +1,27 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize,
7 db: any
8}): Promise<void> {
9 {
10 const data = {
11 type: Sequelize.STRING,
12 allowNull: true,
13 defaultValue: null
14 }
15
16 await utils.queryInterface.addColumn('videoCaption', 'fileUrl', data)
17 }
18}
19
20function down (options) {
21 throw new Error('Not implemented.')
22}
23
24export {
25 up,
26 down
27}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 7a9d5168b..6bc2258cc 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -6,7 +6,8 @@ import {
6 ActivityHashTagObject, 6 ActivityHashTagObject,
7 ActivityMagnetUrlObject, 7 ActivityMagnetUrlObject,
8 ActivityPlaylistSegmentHashesObject, 8 ActivityPlaylistSegmentHashesObject,
9 ActivityPlaylistUrlObject, ActivityTagObject, 9 ActivityPlaylistUrlObject,
10 ActivityTagObject,
10 ActivityUrlObject, 11 ActivityUrlObject,
11 ActivityVideoUrlObject, 12 ActivityVideoUrlObject,
12 VideoState 13 VideoState
@@ -17,14 +18,14 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validat
17import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 18import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
18import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 19import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
19import { logger } from '../../helpers/logger' 20import { logger } from '../../helpers/logger'
20import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 21import { doRequest } from '../../helpers/requests'
21import { 22import {
22 ACTIVITY_PUB, 23 ACTIVITY_PUB,
23 MIMETYPES, 24 MIMETYPES,
24 P2P_MEDIA_LOADER_PEER_VERSION, 25 P2P_MEDIA_LOADER_PEER_VERSION,
25 PREVIEWS_SIZE, 26 PREVIEWS_SIZE,
26 REMOTE_SCHEME, 27 REMOTE_SCHEME,
27 STATIC_PATHS 28 STATIC_PATHS, THUMBNAILS_SIZE
28} from '../../initializers/constants' 29} from '../../initializers/constants'
29import { TagModel } from '../../models/video/tag' 30import { TagModel } from '../../models/video/tag'
30import { VideoModel } from '../../models/video/video' 31import { VideoModel } from '../../models/video/video'
@@ -40,7 +41,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub
40import { createRates } from './video-rates' 41import { createRates } from './video-rates'
41import { addVideoShares, shareVideoByServerAndChannel } from './share' 42import { addVideoShares, shareVideoByServerAndChannel } from './share'
42import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' 43import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
43import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 44import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
44import { Notifier } from '../notifier' 45import { Notifier } from '../notifier'
45import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' 46import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
46import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 47import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@@ -71,6 +72,7 @@ import {
71 MVideoThumbnail 72 MVideoThumbnail
72} from '../../typings/models' 73} from '../../typings/models'
73import { MThumbnail } from '../../typings/models/video/thumbnail' 74import { MThumbnail } from '../../typings/models/video/thumbnail'
75import { maxBy, minBy } from 'lodash'
74 76
75async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) { 77async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
76 const video = videoArg as MVideoAP 78 const video = videoArg as MVideoAP
@@ -131,19 +133,6 @@ async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
131 return body.description ? body.description : '' 133 return body.description ? body.description : ''
132} 134}
133 135
134function fetchRemoteVideoStaticFile (video: MVideoAccountLight, path: string, destPath: string) {
135 const url = buildRemoteBaseUrl(video, path)
136
137 // We need to provide a callback, if no we could have an uncaught exception
138 return doRequestAndSaveToFile({ uri: url }, destPath)
139}
140
141function buildRemoteBaseUrl (video: MVideoAccountLight, path: string) {
142 const host = video.VideoChannel.Account.Actor.Server.host
143
144 return REMOTE_SCHEME.HTTP + '://' + host + path
145}
146
147function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { 136function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
148 const channel = videoObject.attributedTo.find(a => a.type === 'Group') 137 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
149 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) 138 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
@@ -173,7 +162,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
173 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate) 162 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
174 163
175 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner) 164 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
176 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) 165 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes }))
177 } else { 166 } else {
178 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) 167 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
179 } 168 }
@@ -183,7 +172,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
183 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate) 172 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
184 173
185 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner) 174 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
186 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) 175 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes }))
187 } else { 176 } else {
188 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) 177 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
189 } 178 }
@@ -193,7 +182,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
193 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) 182 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
194 183
195 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner) 184 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
196 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) 185 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
197 } else { 186 } else {
198 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) 187 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
199 } 188 }
@@ -203,7 +192,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
203 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) 192 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
204 193
205 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner) 194 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
206 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) 195 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
207 } else { 196 } else {
208 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' }) 197 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
209 } 198 }
@@ -284,7 +273,7 @@ async function updateVideoFromAP (options: {
284 let thumbnailModel: MThumbnail 273 let thumbnailModel: MThumbnail
285 274
286 try { 275 try {
287 thumbnailModel = await createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE) 276 thumbnailModel = await createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
288 } catch (err) { 277 } catch (err) {
289 logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }) 278 logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
290 } 279 }
@@ -327,8 +316,7 @@ async function updateVideoFromAP (options: {
327 316
328 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t) 317 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
329 318
330 // FIXME: use icon URL instead 319 const previewUrl = videoUpdated.getPreview().getFileUrl(videoUpdated)
331 const previewUrl = buildRemoteBaseUrl(videoUpdated, join(STATIC_PATHS.PREVIEWS, videoUpdated.getPreview().filename))
332 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) 320 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
333 await videoUpdated.addAndSaveThumbnail(previewModel, t) 321 await videoUpdated.addAndSaveThumbnail(previewModel, t)
334 322
@@ -391,7 +379,7 @@ async function updateVideoFromAP (options: {
391 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t) 379 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
392 380
393 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { 381 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
394 return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, t) 382 return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, c.url, t)
395 }) 383 })
396 await Promise.all(videoCaptionsPromises) 384 await Promise.all(videoCaptionsPromises)
397 } 385 }
@@ -483,7 +471,6 @@ export {
483 federateVideoIfNeeded, 471 federateVideoIfNeeded,
484 fetchRemoteVideo, 472 fetchRemoteVideo,
485 getOrCreateVideoAndAccountAndChannel, 473 getOrCreateVideoAndAccountAndChannel,
486 fetchRemoteVideoStaticFile,
487 fetchRemoteVideoDescription, 474 fetchRemoteVideoDescription,
488 getOrCreateVideoChannelFromVideoObject 475 getOrCreateVideoChannelFromVideoObject
489} 476}
@@ -519,7 +506,7 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
519 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to) 506 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
520 const video = VideoModel.build(videoData) as MVideoThumbnail 507 const video = VideoModel.build(videoData) as MVideoThumbnail
521 508
522 const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE) 509 const promiseThumbnail = createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
523 510
524 let thumbnailModel: MThumbnail 511 let thumbnailModel: MThumbnail
525 if (waitThumbnail === true) { 512 if (waitThumbnail === true) {
@@ -534,9 +521,12 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
534 521
535 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) 522 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
536 523
537 // FIXME: use icon URL instead 524 const previewIcon = getPreviewFromIcons(videoObject)
538 const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName())) 525 const previewUrl = previewIcon
539 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) 526 ? previewIcon.url
527 : buildRemoteVideoBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
528 const previewModel = createPlaceholderThumbnail(previewUrl, videoCreated, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
529
540 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t) 530 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
541 531
542 // Process files 532 // Process files
@@ -567,7 +557,7 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
567 557
568 // Process captions 558 // Process captions
569 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { 559 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
570 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) 560 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, c.url, t)
571 }) 561 })
572 await Promise.all(videoCaptionsPromises) 562 await Promise.all(videoCaptionsPromises)
573 563
@@ -721,3 +711,19 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec
721 711
722 return attributes 712 return attributes
723} 713}
714
715function getThumbnailFromIcons (videoObject: VideoTorrentObject) {
716 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
717 // Fallback if there are not valid icons
718 if (validIcons.length === 0) validIcons = videoObject.icon
719
720 return minBy(validIcons, 'width')
721}
722
723function getPreviewFromIcons (videoObject: VideoTorrentObject) {
724 const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
725
726 // FIXME: don't put a fallback here for compatibility with PeerTube <2.2
727
728 return maxBy(validIcons, 'width')
729}
diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts
index 440c3fde8..26ab3bd0d 100644
--- a/server/lib/files-cache/videos-caption-cache.ts
+++ b/server/lib/files-cache/videos-caption-cache.ts
@@ -5,7 +5,7 @@ import { VideoCaptionModel } from '../../models/video/video-caption'
5import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 5import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
6import { CONFIG } from '../../initializers/config' 6import { CONFIG } from '../../initializers/config'
7import { logger } from '../../helpers/logger' 7import { logger } from '../../helpers/logger'
8import { fetchRemoteVideoStaticFile } from '../activitypub' 8import { doRequestAndSaveToFile } from '@server/helpers/requests'
9 9
10type GetPathParam = { videoId: string, language: string } 10type GetPathParam = { videoId: string, language: string }
11 11
@@ -46,11 +46,10 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
46 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) 46 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
47 if (!video) return undefined 47 if (!video) return undefined
48 48
49 // FIXME: use URL 49 const remoteUrl = videoCaption.getFileUrl(video)
50 const remoteStaticPath = videoCaption.getCaptionStaticPath()
51 const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) 50 const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
52 51
53 await fetchRemoteVideoStaticFile(video, remoteStaticPath, destPath) 52 await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
54 53
55 return { isOwned: false, path: destPath } 54 return { isOwned: false, path: destPath }
56 } 55 }
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts
index 3da6bb138..7bfeb5783 100644
--- a/server/lib/files-cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/videos-preview-cache.ts
@@ -2,8 +2,8 @@ import { join } from 'path'
2import { FILES_CACHE, STATIC_PATHS } from '../../initializers/constants' 2import { FILES_CACHE, STATIC_PATHS } from '../../initializers/constants'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
5import { CONFIG } from '../../initializers/config' 5import { doRequestAndSaveToFile } from '@server/helpers/requests'
6import { fetchRemoteVideoStaticFile } from '../activitypub' 6import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
7 7
8class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { 8class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
9 9
@@ -32,11 +32,11 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
32 32
33 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') 33 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
34 34
35 // FIXME: use URL 35 const preview = video.getPreview()
36 const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreview().filename) 36 const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename)
37 const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreview().filename)
38 37
39 await fetchRemoteVideoStaticFile(video, remoteStaticPath, destPath) 38 const remoteUrl = preview.getFileUrl(video)
39 await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
40 40
41 return { isOwned: false, path: destPath } 41 return { isOwned: false, path: destPath }
42 } 42 }
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
index 73fa5ed04..2258cd029 100644
--- a/server/lib/job-queue/handlers/video-views.ts
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -23,6 +23,8 @@ async function processVideosViews () {
23 for (const videoId of videoIds) { 23 for (const videoId of videoIds) {
24 try { 24 try {
25 const views = await Redis.Instance.getVideoViews(videoId, hour) 25 const views = await Redis.Instance.getVideoViews(videoId, hour)
26 await Redis.Instance.deleteVideoViews(videoId, hour)
27
26 if (views) { 28 if (views) {
27 logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) 29 logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
28 30
@@ -52,8 +54,6 @@ async function processVideosViews () {
52 logger.error('Cannot create video views for video %d in hour %d.', videoId, hour, { err }) 54 logger.error('Cannot create video views for video %d in hour %d.', videoId, hour, { err })
53 } 55 }
54 } 56 }
55
56 await Redis.Instance.deleteVideoViews(videoId, hour)
57 } catch (err) { 57 } catch (err) {
58 logger.error('Cannot update video views of video %d in hour %d.', videoId, hour, { err }) 58 logger.error('Cannot update video views of video %d in hour %d.', videoId, hour, { err })
59 } 59 }
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index a1c623b25..61f07c487 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -136,7 +136,6 @@ class JobQueue {
136 136
137 const filteredJobTypes = this.filterJobTypes(jobType) 137 const filteredJobTypes = this.filterJobTypes(jobType)
138 138
139 // TODO: optimize
140 for (const jobType of filteredJobTypes) { 139 for (const jobType of filteredJobTypes) {
141 const queue = this.queues[ jobType ] 140 const queue = this.queues[ jobType ]
142 if (queue === undefined) { 141 if (queue === undefined) {
diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts
index 3b011b1d2..b69bc0872 100644
--- a/server/models/video/thumbnail.ts
+++ b/server/models/video/thumbnail.ts
@@ -19,6 +19,8 @@ import { CONFIG } from '../../initializers/config'
19import { VideoModel } from './video' 19import { VideoModel } from './video'
20import { VideoPlaylistModel } from './video-playlist' 20import { VideoPlaylistModel } from './video-playlist'
21import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' 21import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
22import { MVideoAccountLight } from '@server/typings/models'
23import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
22 24
23@Table({ 25@Table({
24 tableName: 'thumbnail', 26 tableName: 'thumbnail',
@@ -126,11 +128,14 @@ export class ThumbnailModel extends Model<ThumbnailModel> {
126 return videoUUID + '.jpg' 128 return videoUUID + '.jpg'
127 } 129 }
128 130
129 getFileUrl (isLocal: boolean) { 131 getFileUrl (video: MVideoAccountLight) {
130 if (isLocal === false) return this.fileUrl 132 const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename
131 133
132 const staticPath = ThumbnailModel.types[this.type].staticPath 134 if (video.isOwned()) return WEBSERVER.URL + staticPath
133 return WEBSERVER.URL + staticPath + this.filename 135 if (this.fileUrl) return this.fileUrl
136
137 // Fallback if we don't have a file URL
138 return buildRemoteVideoBaseUrl(video, staticPath)
134 } 139 }
135 140
136 getPath () { 141 getPath () {
diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts
index 6335d44e4..1307c27f1 100644
--- a/server/models/video/video-caption.ts
+++ b/server/models/video/video-caption.ts
@@ -4,7 +4,7 @@ import {
4 BeforeDestroy, 4 BeforeDestroy,
5 BelongsTo, 5 BelongsTo,
6 Column, 6 Column,
7 CreatedAt, 7 CreatedAt, DataType,
8 ForeignKey, 8 ForeignKey,
9 Is, 9 Is,
10 Model, 10 Model,
@@ -16,13 +16,14 @@ import { buildWhereIdOrUUID, throwIfNotValid } from '../utils'
16import { VideoModel } from './video' 16import { VideoModel } from './video'
17import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' 17import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions'
18import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' 18import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
19import { LAZY_STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants' 19import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, VIDEO_LANGUAGES, WEBSERVER } from '../../initializers/constants'
20import { join } from 'path' 20import { join } from 'path'
21import { logger } from '../../helpers/logger' 21import { logger } from '../../helpers/logger'
22import { remove } from 'fs-extra' 22import { remove } from 'fs-extra'
23import { CONFIG } from '../../initializers/config' 23import { CONFIG } from '../../initializers/config'
24import * as Bluebird from 'bluebird' 24import * as Bluebird from 'bluebird'
25import { MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/typings/models' 25import { MVideo, MVideoAccountLight, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/typings/models'
26import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
26 27
27export enum ScopeNames { 28export enum ScopeNames {
28 WITH_VIDEO_UUID_AND_REMOTE = 'WITH_VIDEO_UUID_AND_REMOTE' 29 WITH_VIDEO_UUID_AND_REMOTE = 'WITH_VIDEO_UUID_AND_REMOTE'
@@ -64,6 +65,10 @@ export class VideoCaptionModel extends Model<VideoCaptionModel> {
64 @Column 65 @Column
65 language: string 66 language: string
66 67
68 @AllowNull(true)
69 @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max))
70 fileUrl: string
71
67 @ForeignKey(() => VideoModel) 72 @ForeignKey(() => VideoModel)
68 @Column 73 @Column
69 videoId: number 74 videoId: number
@@ -114,10 +119,11 @@ export class VideoCaptionModel extends Model<VideoCaptionModel> {
114 return VideoCaptionModel.findOne(query) 119 return VideoCaptionModel.findOne(query)
115 } 120 }
116 121
117 static insertOrReplaceLanguage (videoId: number, language: string, transaction: Transaction) { 122 static insertOrReplaceLanguage (videoId: number, language: string, fileUrl: string, transaction: Transaction) {
118 const values = { 123 const values = {
119 videoId, 124 videoId,
120 language 125 language,
126 fileUrl
121 } 127 }
122 128
123 return VideoCaptionModel.upsert(values, { transaction, returning: true }) 129 return VideoCaptionModel.upsert(values, { transaction, returning: true })
@@ -175,4 +181,14 @@ export class VideoCaptionModel extends Model<VideoCaptionModel> {
175 removeCaptionFile (this: MVideoCaptionFormattable) { 181 removeCaptionFile (this: MVideoCaptionFormattable) {
176 return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.getCaptionName()) 182 return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.getCaptionName())
177 } 183 }
184
185 getFileUrl (video: MVideoAccountLight) {
186 if (!this.Video) this.Video = video as VideoModel
187
188 if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath()
189 if (this.fileUrl) return this.fileUrl
190
191 // Fallback if we don't have a file URL
192 return buildRemoteVideoBaseUrl(video, this.getCaptionStaticPath())
193 }
178} 194}
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index 2aa5b8677..bb50edcaa 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -307,11 +307,12 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoTorrentObject {
307 for (const caption of video.VideoCaptions) { 307 for (const caption of video.VideoCaptions) {
308 subtitleLanguage.push({ 308 subtitleLanguage.push({
309 identifier: caption.language, 309 identifier: caption.language,
310 name: VideoCaptionModel.getLanguageLabel(caption.language) 310 name: VideoCaptionModel.getLanguageLabel(caption.language),
311 url: caption.getFileUrl(video)
311 }) 312 })
312 } 313 }
313 314
314 const miniature = video.getMiniature() 315 const icons = [ video.getMiniature(), video.getPreview() ]
315 316
316 return { 317 return {
317 type: 'Video' as 'Video', 318 type: 'Video' as 'Video',
@@ -336,13 +337,13 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoTorrentObject {
336 content: video.getTruncatedDescription(), 337 content: video.getTruncatedDescription(),
337 support: video.support, 338 support: video.support,
338 subtitleLanguage, 339 subtitleLanguage,
339 icon: { 340 icon: icons.map(i => ({
340 type: 'Image', 341 type: 'Image',
341 url: miniature.getFileUrl(video.isOwned()), 342 url: i.getFileUrl(video),
342 mediaType: 'image/jpeg', 343 mediaType: 'image/jpeg',
343 width: miniature.width, 344 width: i.width,
344 height: miniature.height 345 height: i.height
345 }, 346 })),
346 url, 347 url,
347 likes: getVideoLikesActivityPubUrl(video), 348 likes: getVideoLikesActivityPubUrl(video),
348 dislikes: getVideoDislikesActivityPubUrl(video), 349 dislikes: getVideoDislikesActivityPubUrl(video),
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 20e1f1c4a..1a924e6c9 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1121,7 +1121,7 @@ export class VideoModel extends Model<VideoModel> {
1121 }, 1121 },
1122 include: [ 1122 include: [
1123 { 1123 {
1124 attributes: [ 'language' ], 1124 attributes: [ 'language', 'fileUrl' ],
1125 model: VideoCaptionModel.unscoped(), 1125 model: VideoCaptionModel.unscoped(),
1126 required: false 1126 required: false
1127 }, 1127 },
diff --git a/server/typings/models/video/video-caption.ts b/server/typings/models/video/video-caption.ts
index ffa56f544..eeddedb40 100644
--- a/server/typings/models/video/video-caption.ts
+++ b/server/typings/models/video/video-caption.ts
@@ -11,6 +11,7 @@ export type MVideoCaption = Omit<VideoCaptionModel, 'Video'>
11// ############################################################################ 11// ############################################################################
12 12
13export type MVideoCaptionLanguage = Pick<MVideoCaption, 'language'> 13export type MVideoCaptionLanguage = Pick<MVideoCaption, 'language'>
14export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'getFileUrl'>
14 15
15export type MVideoCaptionVideo = MVideoCaption & 16export type MVideoCaptionVideo = MVideoCaption &
16 Use<'Video', Pick<MVideo, 'id' | 'remote' | 'uuid'>> 17 Use<'Video', Pick<MVideo, 'id' | 'remote' | 'uuid'>>
diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts
index bcc5e5028..82d76f40c 100644
--- a/server/typings/models/video/video.ts
+++ b/server/typings/models/video/video.ts
@@ -9,7 +9,7 @@ import {
9 MChannelUserId 9 MChannelUserId
10} from './video-channels' 10} from './video-channels'
11import { MTag } from './tag' 11import { MTag } from './tag'
12import { MVideoCaptionLanguage } from './video-caption' 12import { MVideoCaptionLanguage, MVideoCaptionLanguageUrl } from './video-caption'
13import { 13import {
14 MStreamingPlaylistFiles, 14 MStreamingPlaylistFiles,
15 MStreamingPlaylistRedundancies, 15 MStreamingPlaylistRedundancies,
@@ -140,7 +140,7 @@ export type MVideoAP = MVideo &
140 Use<'Tags', MTag[]> & 140 Use<'Tags', MTag[]> &
141 Use<'VideoChannel', MChannelAccountLight> & 141 Use<'VideoChannel', MChannelAccountLight> &
142 Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & 142 Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> &
143 Use<'VideoCaptions', MVideoCaptionLanguage[]> & 143 Use<'VideoCaptions', MVideoCaptionLanguageUrl[]> &
144 Use<'VideoBlacklist', MVideoBlacklistUnfederated> & 144 Use<'VideoBlacklist', MVideoBlacklistUnfederated> &
145 Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & 145 Use<'VideoFiles', MVideoFileRedundanciesOpt[]> &
146 Use<'Thumbnails', MThumbnail[]> 146 Use<'Thumbnails', MThumbnail[]>