aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-02-04 15:00:47 +0100
committerChocobozzz <me@florianbigard.com>2020-02-04 15:00:47 +0100
commit7eba5e1fa81c8e54cb8fe298a96e8070afa50921 (patch)
treea6bd4b13dc0d65addfa82fcf200f2d1853a0723a
parente436baf0b00b3ecf3731aeba02437ebe4906ac5f (diff)
downloadPeerTube-7eba5e1fa81c8e54cb8fe298a96e8070afa50921.tar.gz
PeerTube-7eba5e1fa81c8e54cb8fe298a96e8070afa50921.tar.zst
PeerTube-7eba5e1fa81c8e54cb8fe298a96e8070afa50921.zip
Add model cache for video
When fetching only immutable attributes
-rw-r--r--server/controllers/activitypub/client.ts20
-rw-r--r--server/helpers/middlewares/videos.ts17
-rw-r--r--server/helpers/video.ts12
-rw-r--r--server/middlewares/validators/videos/videos.ts5
-rw-r--r--server/models/model-cache.ts39
-rw-r--r--server/models/video/video.ts30
-rw-r--r--server/typings/express.ts3
-rw-r--r--server/typings/models/video/video.ts1
8 files changed, 106 insertions, 21 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 2812bfe1e..9a5fd6084 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -37,7 +37,7 @@ import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike'
37import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' 37import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists'
38import { VideoPlaylistModel } from '../../models/video/video-playlist' 38import { VideoPlaylistModel } from '../../models/video/video-playlist'
39import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' 39import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
40import { MAccountId, MActorId, MVideo, MVideoAPWithoutCaption } from '@server/typings/models' 40import { MAccountId, MActorId, MVideo, MVideoAPWithoutCaption, MVideoId } from '@server/typings/models'
41 41
42const activityPubClientRouter = express.Router() 42const activityPubClientRouter = express.Router()
43 43
@@ -85,7 +85,7 @@ activityPubClientRouter.get('/videos/watch/:id/activity',
85) 85)
86activityPubClientRouter.get('/videos/watch/:id/announces', 86activityPubClientRouter.get('/videos/watch/:id/announces',
87 executeIfActivityPub, 87 executeIfActivityPub,
88 asyncMiddleware(videosCustomGetValidator('only-video')), 88 asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')),
89 asyncMiddleware(videoAnnouncesController) 89 asyncMiddleware(videoAnnouncesController)
90) 90)
91activityPubClientRouter.get('/videos/watch/:id/announces/:actorId', 91activityPubClientRouter.get('/videos/watch/:id/announces/:actorId',
@@ -95,17 +95,17 @@ activityPubClientRouter.get('/videos/watch/:id/announces/:actorId',
95) 95)
96activityPubClientRouter.get('/videos/watch/:id/likes', 96activityPubClientRouter.get('/videos/watch/:id/likes',
97 executeIfActivityPub, 97 executeIfActivityPub,
98 asyncMiddleware(videosCustomGetValidator('only-video')), 98 asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')),
99 asyncMiddleware(videoLikesController) 99 asyncMiddleware(videoLikesController)
100) 100)
101activityPubClientRouter.get('/videos/watch/:id/dislikes', 101activityPubClientRouter.get('/videos/watch/:id/dislikes',
102 executeIfActivityPub, 102 executeIfActivityPub,
103 asyncMiddleware(videosCustomGetValidator('only-video')), 103 asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')),
104 asyncMiddleware(videoDislikesController) 104 asyncMiddleware(videoDislikesController)
105) 105)
106activityPubClientRouter.get('/videos/watch/:id/comments', 106activityPubClientRouter.get('/videos/watch/:id/comments',
107 executeIfActivityPub, 107 executeIfActivityPub,
108 asyncMiddleware(videosCustomGetValidator('only-video')), 108 asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')),
109 asyncMiddleware(videoCommentsController) 109 asyncMiddleware(videoCommentsController)
110) 110)
111activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId', 111activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId',
@@ -238,7 +238,7 @@ async function videoAnnounceController (req: express.Request, res: express.Respo
238} 238}
239 239
240async function videoAnnouncesController (req: express.Request, res: express.Response) { 240async function videoAnnouncesController (req: express.Request, res: express.Response) {
241 const video = res.locals.onlyVideo 241 const video = res.locals.onlyImmutableVideo
242 242
243 const handler = async (start: number, count: number) => { 243 const handler = async (start: number, count: number) => {
244 const result = await VideoShareModel.listAndCountByVideoId(video.id, start, count) 244 const result = await VideoShareModel.listAndCountByVideoId(video.id, start, count)
@@ -253,21 +253,21 @@ async function videoAnnouncesController (req: express.Request, res: express.Resp
253} 253}
254 254
255async function videoLikesController (req: express.Request, res: express.Response) { 255async function videoLikesController (req: express.Request, res: express.Response) {
256 const video = res.locals.onlyVideo 256 const video = res.locals.onlyImmutableVideo
257 const json = await videoRates(req, 'like', video, getVideoLikesActivityPubUrl(video)) 257 const json = await videoRates(req, 'like', video, getVideoLikesActivityPubUrl(video))
258 258
259 return activityPubResponse(activityPubContextify(json), res) 259 return activityPubResponse(activityPubContextify(json), res)
260} 260}
261 261
262async function videoDislikesController (req: express.Request, res: express.Response) { 262async function videoDislikesController (req: express.Request, res: express.Response) {
263 const video = res.locals.onlyVideo 263 const video = res.locals.onlyImmutableVideo
264 const json = await videoRates(req, 'dislike', video, getVideoDislikesActivityPubUrl(video)) 264 const json = await videoRates(req, 'dislike', video, getVideoDislikesActivityPubUrl(video))
265 265
266 return activityPubResponse(activityPubContextify(json), res) 266 return activityPubResponse(activityPubContextify(json), res)
267} 267}
268 268
269async function videoCommentsController (req: express.Request, res: express.Response) { 269async function videoCommentsController (req: express.Request, res: express.Response) {
270 const video = res.locals.onlyVideo 270 const video = res.locals.onlyImmutableVideo
271 271
272 const handler = async (start: number, count: number) => { 272 const handler = async (start: number, count: number) => {
273 const result = await VideoCommentModel.listAndCountByVideoId(video.id, start, count) 273 const result = await VideoCommentModel.listAndCountByVideoId(video.id, start, count)
@@ -386,7 +386,7 @@ async function actorPlaylists (req: express.Request, account: MAccountId) {
386 return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page) 386 return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page)
387} 387}
388 388
389function videoRates (req: express.Request, rateType: VideoRateType, video: MVideo, url: string) { 389function videoRates (req: express.Request, rateType: VideoRateType, video: MVideoId, url: string) {
390 const handler = async (start: number, count: number) => { 390 const handler = async (start: number, count: number) => {
391 const result = await AccountVideoRateModel.listAndCountAccountUrlsByVideoId(rateType, video.id, start, count) 391 const result = await AccountVideoRateModel.listAndCountAccountUrlsByVideoId(rateType, video.id, start, count)
392 return { 392 return {
diff --git a/server/helpers/middlewares/videos.ts b/server/helpers/middlewares/videos.ts
index 74f529804..409f78650 100644
--- a/server/helpers/middlewares/videos.ts
+++ b/server/helpers/middlewares/videos.ts
@@ -2,7 +2,16 @@ import { Response } from 'express'
2import { fetchVideo, VideoFetchType } from '../video' 2import { fetchVideo, VideoFetchType } from '../video'
3import { UserRight } from '../../../shared/models/users' 3import { UserRight } from '../../../shared/models/users'
4import { VideoChannelModel } from '../../models/video/video-channel' 4import { VideoChannelModel } from '../../models/video/video-channel'
5import { MUser, MUserAccountId, MVideoAccountLight, MVideoFullLight, MVideoThumbnail, MVideoWithRights } from '@server/typings/models' 5import {
6 MUser,
7 MUserAccountId,
8 MVideoAccountLight,
9 MVideoFullLight,
10 MVideoIdThumbnail,
11 MVideoImmutable,
12 MVideoThumbnail,
13 MVideoWithRights
14} from '@server/typings/models'
6 15
7async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') { 16async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') {
8 const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined 17 const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined
@@ -22,8 +31,12 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi
22 res.locals.videoAll = video as MVideoFullLight 31 res.locals.videoAll = video as MVideoFullLight
23 break 32 break
24 33
34 case 'only-immutable-attributes':
35 res.locals.onlyImmutableVideo = video as MVideoImmutable
36 break
37
25 case 'id': 38 case 'id':
26 res.locals.videoId = video 39 res.locals.videoId = video as MVideoIdThumbnail
27 break 40 break
28 41
29 case 'only-video': 42 case 'only-video':
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 5b9c026b1..907564703 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -5,13 +5,15 @@ import {
5 MVideoFullLight, 5 MVideoFullLight,
6 MVideoIdThumbnail, 6 MVideoIdThumbnail,
7 MVideoThumbnail, 7 MVideoThumbnail,
8 MVideoWithRights 8 MVideoWithRights,
9 MVideoImmutable
9} from '@server/typings/models' 10} from '@server/typings/models'
10import { Response } from 'express' 11import { Response } from 'express'
11 12
12type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' 13type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes'
13 14
14function fetchVideo (id: number | string, fetchType: 'all', userId?: number): Bluebird<MVideoFullLight> 15function fetchVideo (id: number | string, fetchType: 'all', userId?: number): Bluebird<MVideoFullLight>
16function fetchVideo (id: number | string, fetchType: 'only-immutable-attributes'): Bluebird<MVideoImmutable>
15function fetchVideo (id: number | string, fetchType: 'only-video', userId?: number): Bluebird<MVideoThumbnail> 17function fetchVideo (id: number | string, fetchType: 'only-video', userId?: number): Bluebird<MVideoThumbnail>
16function fetchVideo (id: number | string, fetchType: 'only-video-with-rights', userId?: number): Bluebird<MVideoWithRights> 18function fetchVideo (id: number | string, fetchType: 'only-video-with-rights', userId?: number): Bluebird<MVideoWithRights>
17function fetchVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Bluebird<MVideoIdThumbnail> 19function fetchVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Bluebird<MVideoIdThumbnail>
@@ -19,14 +21,16 @@ function fetchVideo (
19 id: number | string, 21 id: number | string,
20 fetchType: VideoFetchType, 22 fetchType: VideoFetchType,
21 userId?: number 23 userId?: number
22): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail> 24): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail | MVideoImmutable>
23function fetchVideo ( 25function fetchVideo (
24 id: number | string, 26 id: number | string,
25 fetchType: VideoFetchType, 27 fetchType: VideoFetchType,
26 userId?: number 28 userId?: number
27): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail> { 29): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail | MVideoImmutable> {
28 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) 30 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
29 31
32 if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id)
33
30 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id) 34 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id)
31 35
32 if (fetchType === 'only-video') return VideoModel.load(id) 36 if (fetchType === 'only-video') return VideoModel.load(id)
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index 11dd02706..c14184b35 100644
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -147,7 +147,10 @@ async function checkVideoFollowConstraints (req: express.Request, res: express.R
147 }) 147 })
148} 148}
149 149
150const videosCustomGetValidator = (fetchType: 'all' | 'only-video' | 'only-video-with-rights', authenticateInQuery = false) => { 150const videosCustomGetValidator = (
151 fetchType: 'all' | 'only-video' | 'only-video-with-rights' | 'only-immutable-attributes',
152 authenticateInQuery = false
153) => {
151 return [ 154 return [
152 param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), 155 param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'),
153 156
diff --git a/server/models/model-cache.ts b/server/models/model-cache.ts
index bfa163b6b..8afe3834f 100644
--- a/server/models/model-cache.ts
+++ b/server/models/model-cache.ts
@@ -6,6 +6,10 @@ type ModelCacheType =
6 'local-account-name' 6 'local-account-name'
7 | 'local-actor-name' 7 | 'local-actor-name'
8 | 'local-actor-url' 8 | 'local-actor-url'
9 | 'video-immutable'
10
11type DeleteKey =
12 'video'
9 13
10class ModelCache { 14class ModelCache {
11 15
@@ -14,7 +18,14 @@ class ModelCache {
14 private readonly localCache: { [id in ModelCacheType]: Map<string, any> } = { 18 private readonly localCache: { [id in ModelCacheType]: Map<string, any> } = {
15 'local-account-name': new Map(), 19 'local-account-name': new Map(),
16 'local-actor-name': new Map(), 20 'local-actor-name': new Map(),
17 'local-actor-url': new Map() 21 'local-actor-url': new Map(),
22 'video-immutable': new Map()
23 }
24
25 private readonly deleteIds: {
26 [deleteKey in DeleteKey]: Map<number, { cacheType: ModelCacheType, key: string }[]>
27 } = {
28 video: new Map()
18 } 29 }
19 30
20 private constructor () { 31 private constructor () {
@@ -29,8 +40,9 @@ class ModelCache {
29 key: string 40 key: string
30 fun: () => Bluebird<T> 41 fun: () => Bluebird<T>
31 whitelist?: () => boolean 42 whitelist?: () => boolean
43 deleteKey?: DeleteKey
32 }) { 44 }) {
33 const { cacheType, key, fun, whitelist } = options 45 const { cacheType, key, fun, whitelist, deleteKey } = options
34 46
35 if (whitelist && whitelist() !== true) return fun() 47 if (whitelist && whitelist() !== true) return fun()
36 48
@@ -42,11 +54,34 @@ class ModelCache {
42 } 54 }
43 55
44 return fun().then(m => { 56 return fun().then(m => {
57 if (!m) return m
58
45 if (!whitelist || whitelist()) cache.set(key, m) 59 if (!whitelist || whitelist()) cache.set(key, m)
46 60
61 if (deleteKey) {
62 const map = this.deleteIds[deleteKey]
63 if (!map.has(m.id)) map.set(m.id, [])
64
65 const a = map.get(m.id)
66 a.push({ cacheType, key })
67 }
68
47 return m 69 return m
48 }) 70 })
49 } 71 }
72
73 invalidateCache (deleteKey: DeleteKey, modelId: number) {
74 const map = this.deleteIds[deleteKey]
75
76 if (!map.has(modelId)) return
77
78 for (const toDelete of map.get(modelId)) {
79 logger.debug('Removing %s -> %d of model cache %s -> %s.', deleteKey, modelId, toDelete.cacheType, toDelete.key)
80 this.localCache[toDelete.cacheType].delete(toDelete.key)
81 }
82
83 map.delete(modelId)
84 }
50} 85}
51 86
52export { 87export {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 1ec8d717e..9e02d163f 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -120,7 +120,7 @@ import {
120 MVideoFormattableDetails, 120 MVideoFormattableDetails,
121 MVideoForUser, 121 MVideoForUser,
122 MVideoFullLight, 122 MVideoFullLight,
123 MVideoIdThumbnail, 123 MVideoIdThumbnail, MVideoImmutable,
124 MVideoThumbnail, 124 MVideoThumbnail,
125 MVideoThumbnailBlacklist, 125 MVideoThumbnailBlacklist,
126 MVideoWithAllFiles, 126 MVideoWithAllFiles,
@@ -132,6 +132,7 @@ import { MThumbnail } from '../../typings/models/video/thumbnail'
132import { VideoFile } from '@shared/models/videos/video-file.model' 132import { VideoFile } from '@shared/models/videos/video-file.model'
133import { getHLSDirectory, getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 133import { getHLSDirectory, getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
134import validator from 'validator' 134import validator from 'validator'
135import { ModelCache } from '@server/models/model-cache'
135 136
136export enum ScopeNames { 137export enum ScopeNames {
137 AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS', 138 AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS',
@@ -1074,6 +1075,11 @@ export class VideoModel extends Model<VideoModel> {
1074 return undefined 1075 return undefined
1075 } 1076 }
1076 1077
1078 @BeforeDestroy
1079 static invalidateCache (instance: VideoModel) {
1080 ModelCache.Instance.invalidateCache('video', instance.id)
1081 }
1082
1077 static listLocal (): Bluebird<MVideoWithAllFiles[]> { 1083 static listLocal (): Bluebird<MVideoWithAllFiles[]> {
1078 const query = { 1084 const query = {
1079 where: { 1085 where: {
@@ -1468,6 +1474,28 @@ export class VideoModel extends Model<VideoModel> {
1468 ]).findOne(options) 1474 ]).findOne(options)
1469 } 1475 }
1470 1476
1477 static loadImmutableAttributes (id: number | string, t?: Transaction): Bluebird<MVideoImmutable> {
1478 const fun = () => {
1479 const where = buildWhereIdOrUUID(id)
1480 const options = {
1481 attributes: [
1482 'id', 'url', 'uuid'
1483 ],
1484 where,
1485 transaction: t
1486 }
1487
1488 return VideoModel.unscoped().findOne(options)
1489 }
1490
1491 return ModelCache.Instance.doCache({
1492 cacheType: 'video-immutable',
1493 key: '' + id,
1494 deleteKey: 'video',
1495 fun
1496 })
1497 }
1498
1471 static loadWithRights (id: number | string, t?: Transaction): Bluebird<MVideoWithRights> { 1499 static loadWithRights (id: number | string, t?: Transaction): Bluebird<MVideoWithRights> {
1472 const where = buildWhereIdOrUUID(id) 1500 const where = buildWhereIdOrUUID(id)
1473 const options = { 1501 const options = {
diff --git a/server/typings/express.ts b/server/typings/express.ts
index 43a9b2c99..f4188bf3d 100644
--- a/server/typings/express.ts
+++ b/server/typings/express.ts
@@ -21,7 +21,7 @@ import {
21} from './models' 21} from './models'
22import { MVideoPlaylistFull, MVideoPlaylistFullSummary } from './models/video/video-playlist' 22import { MVideoPlaylistFull, MVideoPlaylistFullSummary } from './models/video/video-playlist'
23import { MVideoImportDefault } from '@server/typings/models/video/video-import' 23import { MVideoImportDefault } from '@server/typings/models/video/video-import'
24import { MAccountBlocklist, MActorUrl, MStreamingPlaylist, MVideoFile } from '@server/typings/models' 24import { MAccountBlocklist, MActorUrl, MStreamingPlaylist, MVideoFile, MVideoImmutable } from '@server/typings/models'
25import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/typings/models/video/video-playlist-element' 25import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/typings/models/video/video-playlist-element'
26import { MAccountVideoRateAccountVideo } from '@server/typings/models/video/video-rate' 26import { MAccountVideoRateAccountVideo } from '@server/typings/models/video/video-rate'
27import { MVideoChangeOwnershipFull } from './models/video/video-change-ownership' 27import { MVideoChangeOwnershipFull } from './models/video/video-change-ownership'
@@ -35,6 +35,7 @@ declare module 'express' {
35 35
36 locals: { 36 locals: {
37 videoAll?: MVideoFullLight 37 videoAll?: MVideoFullLight
38 onlyImmutableVideo?: MVideoImmutable
38 onlyVideo?: MVideoThumbnail 39 onlyVideo?: MVideoThumbnail
39 onlyVideoWithRights?: MVideoWithRights 40 onlyVideoWithRights?: MVideoWithRights
40 videoId?: MVideoIdThumbnail 41 videoId?: MVideoIdThumbnail
diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts
index 7eff0a913..3ebb5a762 100644
--- a/server/typings/models/video/video.ts
+++ b/server/typings/models/video/video.ts
@@ -37,6 +37,7 @@ export type MVideoId = Pick<MVideo, 'id'>
37export type MVideoUrl = Pick<MVideo, 'url'> 37export type MVideoUrl = Pick<MVideo, 'url'>
38export type MVideoUUID = Pick<MVideo, 'uuid'> 38export type MVideoUUID = Pick<MVideo, 'uuid'>
39 39
40export type MVideoImmutable = Pick<MVideo, 'id' | 'url' | 'uuid'>
40export type MVideoIdUrl = MVideoId & MVideoUrl 41export type MVideoIdUrl = MVideoId & MVideoUrl
41export type MVideoFeed = Pick<MVideo, 'name' | 'uuid'> 42export type MVideoFeed = Pick<MVideo, 'name' | 'uuid'>
42 43