aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-02-04 15:45:41 +0100
committerChocobozzz <me@florianbigard.com>2020-02-04 15:58:59 +0100
commit943e5193905908dd1f2800d8810c635d86e3b28f (patch)
tree961973733e6e4afb58ac222d2847a3fc4b6d6d60 /server
parent7eba5e1fa81c8e54cb8fe298a96e8070afa50921 (diff)
downloadPeerTube-943e5193905908dd1f2800d8810c635d86e3b28f.tar.gz
PeerTube-943e5193905908dd1f2800d8810c635d86e3b28f.tar.zst
PeerTube-943e5193905908dd1f2800d8810c635d86e3b28f.zip
Don't refresh videos when processing views
It allows us to use a cache
Diffstat (limited to 'server')
-rw-r--r--server/helpers/video.ts15
-rw-r--r--server/lib/activitypub/audience.ts12
-rw-r--r--server/lib/activitypub/process/process-view.ts3
-rw-r--r--server/lib/activitypub/send/utils.ts4
-rw-r--r--server/lib/activitypub/videos.ts46
-rw-r--r--server/middlewares/validators/videos/videos.ts3
-rw-r--r--server/models/model-cache.ts6
-rw-r--r--server/models/video/video.ts36
-rw-r--r--server/typings/models/video/video.ts2
9 files changed, 94 insertions, 33 deletions
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 907564703..4fe2a60f0 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -38,14 +38,23 @@ function fetchVideo (
38 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) 38 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
39} 39}
40 40
41type VideoFetchByUrlType = 'all' | 'only-video' 41type VideoFetchByUrlType = 'all' | 'only-video' | 'only-immutable-attributes'
42 42
43function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles> 43function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles>
44function fetchVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Bluebird<MVideoImmutable>
44function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail> 45function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail>
45function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> 46function fetchVideoByUrl (
46function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> { 47 url: string,
48 fetchType: VideoFetchByUrlType
49): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable>
50function fetchVideoByUrl (
51 url: string,
52 fetchType: VideoFetchByUrlType
53): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
47 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) 54 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
48 55
56 if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
57
49 if (fetchType === 'only-video') return VideoModel.loadByUrl(url) 58 if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
50} 59}
51 60
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts
index f2ab54cf7..39caeef7b 100644
--- a/server/lib/activitypub/audience.ts
+++ b/server/lib/activitypub/audience.ts
@@ -4,7 +4,15 @@ import { ACTIVITY_PUB } from '../../initializers/constants'
4import { ActorModel } from '../../models/activitypub/actor' 4import { ActorModel } from '../../models/activitypub/actor'
5import { VideoModel } from '../../models/video/video' 5import { VideoModel } from '../../models/video/video'
6import { VideoShareModel } from '../../models/video/video-share' 6import { VideoShareModel } from '../../models/video/video-share'
7import { MActorFollowersUrl, MActorLight, MCommentOwner, MCommentOwnerVideo, MVideo, MVideoAccountLight } from '../../typings/models' 7import {
8 MActorFollowersUrl,
9 MActorLight,
10 MCommentOwner,
11 MCommentOwnerVideo,
12 MVideo,
13 MVideoAccountLight,
14 MVideoId
15} from '../../typings/models'
8 16
9function getRemoteVideoAudience (video: MVideoAccountLight, actorsInvolvedInVideo: MActorFollowersUrl[]): ActivityAudience { 17function getRemoteVideoAudience (video: MVideoAccountLight, actorsInvolvedInVideo: MActorFollowersUrl[]): ActivityAudience {
10 return { 18 return {
@@ -48,7 +56,7 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[
48 } 56 }
49} 57}
50 58
51async function getActorsInvolvedInVideo (video: MVideo, t: Transaction) { 59async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) {
52 const actors: MActorLight[] = await VideoShareModel.loadActorsByShare(video.id, t) 60 const actors: MActorLight[] = await VideoShareModel.loadActorsByShare(video.id, t)
53 61
54 const videoAll = video as VideoModel 62 const videoAll = video as VideoModel
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
index df29ee968..b3b6c933d 100644
--- a/server/lib/activitypub/process/process-view.ts
+++ b/server/lib/activitypub/process/process-view.ts
@@ -23,7 +23,8 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct
23 23
24 const options = { 24 const options = {
25 videoObject, 25 videoObject,
26 fetchType: 'only-video' as 'only-video' 26 fetchType: 'only-immutable-attributes' as 'only-immutable-attributes',
27 allowRefresh: false as false
27 } 28 }
28 const { video } = await getOrCreateVideoAndAccountAndChannel(options) 29 const { video } = await getOrCreateVideoAndAccountAndChannel(options)
29 30
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts
index 0d67bb3d6..9436daf17 100644
--- a/server/lib/activitypub/send/utils.ts
+++ b/server/lib/activitypub/send/utils.ts
@@ -7,7 +7,7 @@ import { JobQueue } from '../../job-queue'
7import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience' 7import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
8import { getServerActor } from '../../../helpers/utils' 8import { getServerActor } from '../../../helpers/utils'
9import { afterCommitIfTransaction } from '../../../helpers/database-utils' 9import { afterCommitIfTransaction } from '../../../helpers/database-utils'
10import { MActorWithInboxes, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight } from '../../../typings/models' 10import { MActorWithInboxes, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../../typings/models'
11import { ContextType } from '@server/helpers/activitypub' 11import { ContextType } from '@server/helpers/activitypub'
12 12
13async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { 13async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
@@ -43,7 +43,7 @@ async function forwardVideoRelatedActivity (
43 activity: Activity, 43 activity: Activity,
44 t: Transaction, 44 t: Transaction,
45 followersException: MActorWithInboxes[] = [], 45 followersException: MActorWithInboxes[] = [],
46 video: MVideo 46 video: MVideoId
47) { 47) {
48 // Mastodon does not add our announces in audience, so we forward to them manually 48 // Mastodon does not add our announces in audience, so we forward to them manually
49 const additionalActors = await getActorsInvolvedInVideo(video, t) 49 const additionalActors = await getActorsInvolvedInVideo(video, t)
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 9e43caa20..7d8296e45 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -68,7 +68,7 @@ import {
68 MVideoAPWithoutCaption, 68 MVideoAPWithoutCaption,
69 MVideoFile, 69 MVideoFile,
70 MVideoFullLight, 70 MVideoFullLight,
71 MVideoId, 71 MVideoId, MVideoImmutable,
72 MVideoThumbnail 72 MVideoThumbnail
73} from '../../typings/models' 73} from '../../typings/models'
74import { MThumbnail } from '../../typings/models/video/thumbnail' 74import { MThumbnail } from '../../typings/models/video/thumbnail'
@@ -200,24 +200,41 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
200 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })) 200 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
201} 201}
202 202
203function getOrCreateVideoAndAccountAndChannel (options: { 203type GetVideoResult <T> = Promise<{
204 video: T
205 created: boolean
206 autoBlacklisted?: boolean
207}>
208
209type GetVideoParamAll = {
204 videoObject: { id: string } | string 210 videoObject: { id: string } | string
205 syncParam?: SyncParam 211 syncParam?: SyncParam
206 fetchType?: 'all' 212 fetchType?: 'all'
207 allowRefresh?: boolean 213 allowRefresh?: boolean
208}): Promise<{ video: MVideoAccountLightBlacklistAllFiles, created: boolean, autoBlacklisted?: boolean }> 214}
209function getOrCreateVideoAndAccountAndChannel (options: { 215
216type GetVideoParamImmutable = {
210 videoObject: { id: string } | string 217 videoObject: { id: string } | string
211 syncParam?: SyncParam 218 syncParam?: SyncParam
212 fetchType?: VideoFetchByUrlType 219 fetchType: 'only-immutable-attributes'
213 allowRefresh?: boolean 220 allowRefresh: false
214}): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> 221}
215async function getOrCreateVideoAndAccountAndChannel (options: { 222
223type GetVideoParamOther = {
216 videoObject: { id: string } | string 224 videoObject: { id: string } | string
217 syncParam?: SyncParam 225 syncParam?: SyncParam
218 fetchType?: VideoFetchByUrlType 226 fetchType?: 'all' | 'only-video'
219 allowRefresh?: boolean // true by default 227 allowRefresh?: boolean
220}): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> { 228}
229
230function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
231function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
232function getOrCreateVideoAndAccountAndChannel (
233 options: GetVideoParamOther
234): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
235async function getOrCreateVideoAndAccountAndChannel (
236 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
237): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
221 // Default params 238 // Default params
222 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } 239 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
223 const fetchType = options.fetchType || 'all' 240 const fetchType = options.fetchType || 'all'
@@ -225,12 +242,13 @@ async function getOrCreateVideoAndAccountAndChannel (options: {
225 242
226 // Get video url 243 // Get video url
227 const videoUrl = getAPId(options.videoObject) 244 const videoUrl = getAPId(options.videoObject)
228
229 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) 245 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
246
230 if (videoFromDatabase) { 247 if (videoFromDatabase) {
231 if (videoFromDatabase.isOutdated() && allowRefresh === true) { 248 // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
249 if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
232 const refreshOptions = { 250 const refreshOptions = {
233 video: videoFromDatabase, 251 video: videoFromDatabase as MVideoThumbnail,
234 fetchedType: fetchType, 252 fetchedType: fetchType,
235 syncParam 253 syncParam
236 } 254 }
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index c14184b35..a027c4840 100644
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -160,6 +160,9 @@ const videosCustomGetValidator = (
160 if (areValidationErrors(req, res)) return 160 if (areValidationErrors(req, res)) return
161 if (!await doesVideoExist(req.params.id, res, fetchType)) return 161 if (!await doesVideoExist(req.params.id, res, fetchType)) return
162 162
163 // Controllers does not need to check video rights
164 if (fetchType === 'only-immutable-attributes') return next()
165
163 const video = getVideoWithAttributes(res) 166 const video = getVideoWithAttributes(res)
164 const videoAll = video as MVideoFullLight 167 const videoAll = video as MVideoFullLight
165 168
diff --git a/server/models/model-cache.ts b/server/models/model-cache.ts
index 8afe3834f..a87f99aa2 100644
--- a/server/models/model-cache.ts
+++ b/server/models/model-cache.ts
@@ -6,7 +6,8 @@ type ModelCacheType =
6 'local-account-name' 6 'local-account-name'
7 | 'local-actor-name' 7 | 'local-actor-name'
8 | 'local-actor-url' 8 | 'local-actor-url'
9 | 'video-immutable' 9 | 'load-video-immutable-id'
10 | 'load-video-immutable-url'
10 11
11type DeleteKey = 12type DeleteKey =
12 'video' 13 'video'
@@ -19,7 +20,8 @@ class ModelCache {
19 'local-account-name': new Map(), 20 'local-account-name': new Map(),
20 'local-actor-name': new Map(), 21 'local-actor-name': new Map(),
21 'local-actor-url': new Map(), 22 'local-actor-url': new Map(),
22 'video-immutable': new Map() 23 'load-video-immutable-id': new Map(),
24 'load-video-immutable-url': new Map()
23 } 25 }
24 26
25 private readonly deleteIds: { 27 private readonly deleteIds: {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 9e02d163f..5964526a9 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -145,6 +145,7 @@ export enum ScopeNames {
145 WITH_USER_HISTORY = 'WITH_USER_HISTORY', 145 WITH_USER_HISTORY = 'WITH_USER_HISTORY',
146 WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', 146 WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
147 WITH_USER_ID = 'WITH_USER_ID', 147 WITH_USER_ID = 'WITH_USER_ID',
148 WITH_IMMUTABLE_ATTRIBUTES = 'WITH_IMMUTABLE_ATTRIBUTES',
148 WITH_THUMBNAILS = 'WITH_THUMBNAILS' 149 WITH_THUMBNAILS = 'WITH_THUMBNAILS'
149} 150}
150 151
@@ -188,6 +189,9 @@ export type AvailableForListIDsOptions = {
188} 189}
189 190
190@Scopes(() => ({ 191@Scopes(() => ({
192 [ScopeNames.WITH_IMMUTABLE_ATTRIBUTES]: {
193 attributes: [ 'id', 'url', 'uuid', 'remote' ]
194 },
191 [ScopeNames.FOR_API]: (options: ForAPIOptions) => { 195 [ScopeNames.FOR_API]: (options: ForAPIOptions) => {
192 const query: FindOptions = { 196 const query: FindOptions = {
193 include: [ 197 include: [
@@ -1476,20 +1480,16 @@ export class VideoModel extends Model<VideoModel> {
1476 1480
1477 static loadImmutableAttributes (id: number | string, t?: Transaction): Bluebird<MVideoImmutable> { 1481 static loadImmutableAttributes (id: number | string, t?: Transaction): Bluebird<MVideoImmutable> {
1478 const fun = () => { 1482 const fun = () => {
1479 const where = buildWhereIdOrUUID(id) 1483 const query = {
1480 const options = { 1484 where: buildWhereIdOrUUID(id),
1481 attributes: [
1482 'id', 'url', 'uuid'
1483 ],
1484 where,
1485 transaction: t 1485 transaction: t
1486 } 1486 }
1487 1487
1488 return VideoModel.unscoped().findOne(options) 1488 return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query)
1489 } 1489 }
1490 1490
1491 return ModelCache.Instance.doCache({ 1491 return ModelCache.Instance.doCache({
1492 cacheType: 'video-immutable', 1492 cacheType: 'load-video-immutable-id',
1493 key: '' + id, 1493 key: '' + id,
1494 deleteKey: 'video', 1494 deleteKey: 'video',
1495 fun 1495 fun
@@ -1559,6 +1559,26 @@ export class VideoModel extends Model<VideoModel> {
1559 return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(query) 1559 return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(query)
1560 } 1560 }
1561 1561
1562 static loadByUrlImmutableAttributes (url: string, transaction?: Transaction): Bluebird<MVideoImmutable> {
1563 const fun = () => {
1564 const query: FindOptions = {
1565 where: {
1566 url
1567 },
1568 transaction
1569 }
1570
1571 return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query)
1572 }
1573
1574 return ModelCache.Instance.doCache({
1575 cacheType: 'load-video-immutable-url',
1576 key: url,
1577 deleteKey: 'video',
1578 fun
1579 })
1580 }
1581
1562 static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Bluebird<MVideoAccountLightBlacklistAllFiles> { 1582 static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Bluebird<MVideoAccountLightBlacklistAllFiles> {
1563 const query: FindOptions = { 1583 const query: FindOptions = {
1564 where: { 1584 where: {
diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts
index 3ebb5a762..022a9566d 100644
--- a/server/typings/models/video/video.ts
+++ b/server/typings/models/video/video.ts
@@ -37,7 +37,7 @@ export type MVideoId = Pick<MVideo, 'id'>
37export type MVideoUrl = Pick<MVideo, 'url'> 37export type MVideoUrl = Pick<MVideo, 'url'>
38export type MVideoUUID = Pick<MVideo, 'uuid'> 38export type MVideoUUID = Pick<MVideo, 'uuid'>
39 39
40export type MVideoImmutable = Pick<MVideo, 'id' | 'url' | 'uuid'> 40export type MVideoImmutable = Pick<MVideo, 'id' | 'url' | 'uuid' | 'remote' | 'isOwned'>
41export type MVideoIdUrl = MVideoId & MVideoUrl 41export type MVideoIdUrl = MVideoId & MVideoUrl
42export type MVideoFeed = Pick<MVideo, 'name' | 'uuid'> 42export type MVideoFeed = Pick<MVideo, 'name' | 'uuid'>
43 43