aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/videos.ts68
-rw-r--r--server/lib/files-cache/videos-caption-cache.ts7
-rw-r--r--server/lib/files-cache/videos-preview-cache.ts12
-rw-r--r--server/lib/job-queue/handlers/video-views.ts4
-rw-r--r--server/lib/job-queue/job-queue.ts1
5 files changed, 48 insertions, 44 deletions
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 7a9d5168b..6bc2258cc 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -6,7 +6,8 @@ import {
6 ActivityHashTagObject, 6 ActivityHashTagObject,
7 ActivityMagnetUrlObject, 7 ActivityMagnetUrlObject,
8 ActivityPlaylistSegmentHashesObject, 8 ActivityPlaylistSegmentHashesObject,
9 ActivityPlaylistUrlObject, ActivityTagObject, 9 ActivityPlaylistUrlObject,
10 ActivityTagObject,
10 ActivityUrlObject, 11 ActivityUrlObject,
11 ActivityVideoUrlObject, 12 ActivityVideoUrlObject,
12 VideoState 13 VideoState
@@ -17,14 +18,14 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validat
17import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 18import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
18import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 19import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
19import { logger } from '../../helpers/logger' 20import { logger } from '../../helpers/logger'
20import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 21import { doRequest } from '../../helpers/requests'
21import { 22import {
22 ACTIVITY_PUB, 23 ACTIVITY_PUB,
23 MIMETYPES, 24 MIMETYPES,
24 P2P_MEDIA_LOADER_PEER_VERSION, 25 P2P_MEDIA_LOADER_PEER_VERSION,
25 PREVIEWS_SIZE, 26 PREVIEWS_SIZE,
26 REMOTE_SCHEME, 27 REMOTE_SCHEME,
27 STATIC_PATHS 28 STATIC_PATHS, THUMBNAILS_SIZE
28} from '../../initializers/constants' 29} from '../../initializers/constants'
29import { TagModel } from '../../models/video/tag' 30import { TagModel } from '../../models/video/tag'
30import { VideoModel } from '../../models/video/video' 31import { VideoModel } from '../../models/video/video'
@@ -40,7 +41,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub
40import { createRates } from './video-rates' 41import { createRates } from './video-rates'
41import { addVideoShares, shareVideoByServerAndChannel } from './share' 42import { addVideoShares, shareVideoByServerAndChannel } from './share'
42import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' 43import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
43import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 44import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
44import { Notifier } from '../notifier' 45import { Notifier } from '../notifier'
45import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' 46import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
46import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 47import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@@ -71,6 +72,7 @@ import {
71 MVideoThumbnail 72 MVideoThumbnail
72} from '../../typings/models' 73} from '../../typings/models'
73import { MThumbnail } from '../../typings/models/video/thumbnail' 74import { MThumbnail } from '../../typings/models/video/thumbnail'
75import { maxBy, minBy } from 'lodash'
74 76
75async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) { 77async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
76 const video = videoArg as MVideoAP 78 const video = videoArg as MVideoAP
@@ -131,19 +133,6 @@ async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
131 return body.description ? body.description : '' 133 return body.description ? body.description : ''
132} 134}
133 135
134function fetchRemoteVideoStaticFile (video: MVideoAccountLight, path: string, destPath: string) {
135 const url = buildRemoteBaseUrl(video, path)
136
137 // We need to provide a callback, if no we could have an uncaught exception
138 return doRequestAndSaveToFile({ uri: url }, destPath)
139}
140
141function buildRemoteBaseUrl (video: MVideoAccountLight, path: string) {
142 const host = video.VideoChannel.Account.Actor.Server.host
143
144 return REMOTE_SCHEME.HTTP + '://' + host + path
145}
146
147function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { 136function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
148 const channel = videoObject.attributedTo.find(a => a.type === 'Group') 137 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
149 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) 138 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
@@ -173,7 +162,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
173 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate) 162 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
174 163
175 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner) 164 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
176 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) 165 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes }))
177 } else { 166 } else {
178 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) 167 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
179 } 168 }
@@ -183,7 +172,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
183 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate) 172 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
184 173
185 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner) 174 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
186 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) 175 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes }))
187 } else { 176 } else {
188 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) 177 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
189 } 178 }
@@ -193,7 +182,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
193 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) 182 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
194 183
195 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner) 184 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
196 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) 185 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
197 } else { 186 } else {
198 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) 187 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
199 } 188 }
@@ -203,7 +192,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
203 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) 192 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
204 193
205 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner) 194 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
206 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) 195 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
207 } else { 196 } else {
208 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' }) 197 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
209 } 198 }
@@ -284,7 +273,7 @@ async function updateVideoFromAP (options: {
284 let thumbnailModel: MThumbnail 273 let thumbnailModel: MThumbnail
285 274
286 try { 275 try {
287 thumbnailModel = await createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE) 276 thumbnailModel = await createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
288 } catch (err) { 277 } catch (err) {
289 logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }) 278 logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
290 } 279 }
@@ -327,8 +316,7 @@ async function updateVideoFromAP (options: {
327 316
328 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t) 317 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
329 318
330 // FIXME: use icon URL instead 319 const previewUrl = videoUpdated.getPreview().getFileUrl(videoUpdated)
331 const previewUrl = buildRemoteBaseUrl(videoUpdated, join(STATIC_PATHS.PREVIEWS, videoUpdated.getPreview().filename))
332 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) 320 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
333 await videoUpdated.addAndSaveThumbnail(previewModel, t) 321 await videoUpdated.addAndSaveThumbnail(previewModel, t)
334 322
@@ -391,7 +379,7 @@ async function updateVideoFromAP (options: {
391 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t) 379 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
392 380
393 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { 381 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
394 return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, t) 382 return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, c.url, t)
395 }) 383 })
396 await Promise.all(videoCaptionsPromises) 384 await Promise.all(videoCaptionsPromises)
397 } 385 }
@@ -483,7 +471,6 @@ export {
483 federateVideoIfNeeded, 471 federateVideoIfNeeded,
484 fetchRemoteVideo, 472 fetchRemoteVideo,
485 getOrCreateVideoAndAccountAndChannel, 473 getOrCreateVideoAndAccountAndChannel,
486 fetchRemoteVideoStaticFile,
487 fetchRemoteVideoDescription, 474 fetchRemoteVideoDescription,
488 getOrCreateVideoChannelFromVideoObject 475 getOrCreateVideoChannelFromVideoObject
489} 476}
@@ -519,7 +506,7 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
519 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to) 506 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
520 const video = VideoModel.build(videoData) as MVideoThumbnail 507 const video = VideoModel.build(videoData) as MVideoThumbnail
521 508
522 const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE) 509 const promiseThumbnail = createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
523 510
524 let thumbnailModel: MThumbnail 511 let thumbnailModel: MThumbnail
525 if (waitThumbnail === true) { 512 if (waitThumbnail === true) {
@@ -534,9 +521,12 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
534 521
535 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) 522 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
536 523
537 // FIXME: use icon URL instead 524 const previewIcon = getPreviewFromIcons(videoObject)
538 const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName())) 525 const previewUrl = previewIcon
539 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) 526 ? previewIcon.url
527 : buildRemoteVideoBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
528 const previewModel = createPlaceholderThumbnail(previewUrl, videoCreated, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
529
540 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t) 530 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
541 531
542 // Process files 532 // Process files
@@ -567,7 +557,7 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
567 557
568 // Process captions 558 // Process captions
569 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { 559 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
570 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) 560 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, c.url, t)
571 }) 561 })
572 await Promise.all(videoCaptionsPromises) 562 await Promise.all(videoCaptionsPromises)
573 563
@@ -721,3 +711,19 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec
721 711
722 return attributes 712 return attributes
723} 713}
714
715function getThumbnailFromIcons (videoObject: VideoTorrentObject) {
716 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
717 // Fallback if there are not valid icons
718 if (validIcons.length === 0) validIcons = videoObject.icon
719
720 return minBy(validIcons, 'width')
721}
722
723function getPreviewFromIcons (videoObject: VideoTorrentObject) {
724 const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
725
726 // FIXME: don't put a fallback here for compatibility with PeerTube <2.2
727
728 return maxBy(validIcons, 'width')
729}
diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts
index 440c3fde8..26ab3bd0d 100644
--- a/server/lib/files-cache/videos-caption-cache.ts
+++ b/server/lib/files-cache/videos-caption-cache.ts
@@ -5,7 +5,7 @@ import { VideoCaptionModel } from '../../models/video/video-caption'
5import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 5import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
6import { CONFIG } from '../../initializers/config' 6import { CONFIG } from '../../initializers/config'
7import { logger } from '../../helpers/logger' 7import { logger } from '../../helpers/logger'
8import { fetchRemoteVideoStaticFile } from '../activitypub' 8import { doRequestAndSaveToFile } from '@server/helpers/requests'
9 9
10type GetPathParam = { videoId: string, language: string } 10type GetPathParam = { videoId: string, language: string }
11 11
@@ -46,11 +46,10 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
46 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) 46 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
47 if (!video) return undefined 47 if (!video) return undefined
48 48
49 // FIXME: use URL 49 const remoteUrl = videoCaption.getFileUrl(video)
50 const remoteStaticPath = videoCaption.getCaptionStaticPath()
51 const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) 50 const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
52 51
53 await fetchRemoteVideoStaticFile(video, remoteStaticPath, destPath) 52 await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
54 53
55 return { isOwned: false, path: destPath } 54 return { isOwned: false, path: destPath }
56 } 55 }
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts
index 3da6bb138..7bfeb5783 100644
--- a/server/lib/files-cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/videos-preview-cache.ts
@@ -2,8 +2,8 @@ import { join } from 'path'
2import { FILES_CACHE, STATIC_PATHS } from '../../initializers/constants' 2import { FILES_CACHE, STATIC_PATHS } from '../../initializers/constants'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
5import { CONFIG } from '../../initializers/config' 5import { doRequestAndSaveToFile } from '@server/helpers/requests'
6import { fetchRemoteVideoStaticFile } from '../activitypub' 6import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
7 7
8class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { 8class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
9 9
@@ -32,11 +32,11 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
32 32
33 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') 33 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
34 34
35 // FIXME: use URL 35 const preview = video.getPreview()
36 const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreview().filename) 36 const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename)
37 const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreview().filename)
38 37
39 await fetchRemoteVideoStaticFile(video, remoteStaticPath, destPath) 38 const remoteUrl = preview.getFileUrl(video)
39 await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
40 40
41 return { isOwned: false, path: destPath } 41 return { isOwned: false, path: destPath }
42 } 42 }
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
index 73fa5ed04..2258cd029 100644
--- a/server/lib/job-queue/handlers/video-views.ts
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -23,6 +23,8 @@ async function processVideosViews () {
23 for (const videoId of videoIds) { 23 for (const videoId of videoIds) {
24 try { 24 try {
25 const views = await Redis.Instance.getVideoViews(videoId, hour) 25 const views = await Redis.Instance.getVideoViews(videoId, hour)
26 await Redis.Instance.deleteVideoViews(videoId, hour)
27
26 if (views) { 28 if (views) {
27 logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) 29 logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
28 30
@@ -52,8 +54,6 @@ async function processVideosViews () {
52 logger.error('Cannot create video views for video %d in hour %d.', videoId, hour, { err }) 54 logger.error('Cannot create video views for video %d in hour %d.', videoId, hour, { err })
53 } 55 }
54 } 56 }
55
56 await Redis.Instance.deleteVideoViews(videoId, hour)
57 } catch (err) { 57 } catch (err) {
58 logger.error('Cannot update video views of video %d in hour %d.', videoId, hour, { err }) 58 logger.error('Cannot update video views of video %d in hour %d.', videoId, hour, { err })
59 } 59 }
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index a1c623b25..61f07c487 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -136,7 +136,6 @@ class JobQueue {
136 136
137 const filteredJobTypes = this.filterJobTypes(jobType) 137 const filteredJobTypes = this.filterJobTypes(jobType)
138 138
139 // TODO: optimize
140 for (const jobType of filteredJobTypes) { 139 for (const jobType of filteredJobTypes) {
141 const queue = this.queues[ jobType ] 140 const queue = this.queues[ jobType ]
142 if (queue === undefined) { 141 if (queue === undefined) {