diff options
author | Chocobozzz <me@florianbigard.com> | 2021-07-22 11:15:17 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2021-07-22 11:48:24 +0200 |
commit | 20bafcb61bee2a9a10a500908850c9a7d5e3c8c5 (patch) | |
tree | 18d0e8eb693b0fce88b21b282ea6f28836763fe6 | |
parent | 13e13377918b65c30b9334920fef4b43e70b964e (diff) | |
download | PeerTube-20bafcb61bee2a9a10a500908850c9a7d5e3c8c5.tar.gz PeerTube-20bafcb61bee2a9a10a500908850c9a7d5e3c8c5.tar.zst PeerTube-20bafcb61bee2a9a10a500908850c9a7d5e3c8c5.zip |
Move apicache in peertube
Allow us to upgrade to node 16
-rw-r--r-- | package.json | 1 | ||||
-rw-r--r-- | server/controllers/activitypub/client.ts | 4 | ||||
-rw-r--r-- | server/controllers/api/server/stats.ts | 4 | ||||
-rw-r--r-- | server/controllers/bots.ts | 4 | ||||
-rw-r--r-- | server/controllers/feeds.ts | 24 | ||||
-rw-r--r-- | server/controllers/static.ts | 12 | ||||
-rw-r--r-- | server/middlewares/cache.ts | 28 | ||||
-rw-r--r-- | server/middlewares/cache/cache.ts | 32 | ||||
-rw-r--r-- | server/middlewares/cache/index.ts | 1 | ||||
-rw-r--r-- | server/middlewares/cache/shared/api-cache.ts | 269 | ||||
-rw-r--r-- | server/middlewares/cache/shared/index.ts | 1 | ||||
-rw-r--r-- | server/middlewares/index.ts | 1 | ||||
-rw-r--r-- | server/tests/feeds/feeds.ts | 40 | ||||
-rw-r--r-- | server/typings/express/index.d.ts | 11 | ||||
-rw-r--r-- | yarn.lock | 5 |
15 files changed, 372 insertions, 65 deletions
diff --git a/package.json b/package.json index 7611ac9ab..bd18d4c64 100644 --- a/package.json +++ b/package.json | |||
@@ -74,7 +74,6 @@ | |||
74 | }, | 74 | }, |
75 | "dependencies": { | 75 | "dependencies": { |
76 | "@uploadx/core": "^4.4.0", | 76 | "@uploadx/core": "^4.4.0", |
77 | "apicache": "1.6.2", | ||
78 | "async": "^3.0.1", | 77 | "async": "^3.0.1", |
79 | "async-lru": "^1.1.1", | 78 | "async-lru": "^1.1.1", |
80 | "bcrypt": "5.0.1", | 79 | "bcrypt": "5.0.1", |
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index d7de1b9bd..bef4bc068 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts | |||
@@ -24,7 +24,7 @@ import { | |||
24 | videosCustomGetValidator, | 24 | videosCustomGetValidator, |
25 | videosShareValidator | 25 | videosShareValidator |
26 | } from '../../middlewares' | 26 | } from '../../middlewares' |
27 | import { cacheRoute } from '../../middlewares/cache' | 27 | import { cacheRoute } from '../../middlewares/cache/cache' |
28 | import { getAccountVideoRateValidatorFactory, videoCommentGetValidator } from '../../middlewares/validators' | 28 | import { getAccountVideoRateValidatorFactory, videoCommentGetValidator } from '../../middlewares/validators' |
29 | import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' | 29 | import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' |
30 | import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' | 30 | import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' |
@@ -77,7 +77,7 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId', | |||
77 | activityPubClientRouter.get( | 77 | activityPubClientRouter.get( |
78 | [ '/videos/watch/:id', '/w/:id' ], | 78 | [ '/videos/watch/:id', '/w/:id' ], |
79 | executeIfActivityPub, | 79 | executeIfActivityPub, |
80 | asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS)), | 80 | cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS), |
81 | asyncMiddleware(videosCustomGetValidator('all')), | 81 | asyncMiddleware(videosCustomGetValidator('all')), |
82 | asyncMiddleware(videoController) | 82 | asyncMiddleware(videoController) |
83 | ) | 83 | ) |
diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index 3aea12450..397702548 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts | |||
@@ -2,12 +2,12 @@ import * as express from 'express' | |||
2 | import { StatsManager } from '@server/lib/stat-manager' | 2 | import { StatsManager } from '@server/lib/stat-manager' |
3 | import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' | 3 | import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' |
4 | import { asyncMiddleware } from '../../../middlewares' | 4 | import { asyncMiddleware } from '../../../middlewares' |
5 | import { cacheRoute } from '../../../middlewares/cache' | 5 | import { cacheRoute } from '../../../middlewares/cache/cache' |
6 | 6 | ||
7 | const statsRouter = express.Router() | 7 | const statsRouter = express.Router() |
8 | 8 | ||
9 | statsRouter.get('/stats', | 9 | statsRouter.get('/stats', |
10 | asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.STATS)), | 10 | cacheRoute(ROUTE_CACHE_LIFETIME.STATS), |
11 | asyncMiddleware(getStats) | 11 | asyncMiddleware(getStats) |
12 | ) | 12 | ) |
13 | 13 | ||
diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts index 9e92063d4..93aa0cf30 100644 --- a/server/controllers/bots.ts +++ b/server/controllers/bots.ts | |||
@@ -5,7 +5,7 @@ import { SitemapStream, streamToPromise } from 'sitemap' | |||
5 | import { VideoModel } from '../models/video/video' | 5 | import { VideoModel } from '../models/video/video' |
6 | import { VideoChannelModel } from '../models/video/video-channel' | 6 | import { VideoChannelModel } from '../models/video/video-channel' |
7 | import { AccountModel } from '../models/account/account' | 7 | import { AccountModel } from '../models/account/account' |
8 | import { cacheRoute } from '../middlewares/cache' | 8 | import { cacheRoute } from '../middlewares/cache/cache' |
9 | import { buildNSFWFilter } from '../helpers/express-utils' | 9 | import { buildNSFWFilter } from '../helpers/express-utils' |
10 | import { truncate } from 'lodash' | 10 | import { truncate } from 'lodash' |
11 | 11 | ||
@@ -14,7 +14,7 @@ const botsRouter = express.Router() | |||
14 | // Special route that add OpenGraph and oEmbed tags | 14 | // Special route that add OpenGraph and oEmbed tags |
15 | // Do not use a template engine for a so little thing | 15 | // Do not use a template engine for a so little thing |
16 | botsRouter.use('/sitemap.xml', | 16 | botsRouter.use('/sitemap.xml', |
17 | asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.SITEMAP)), | 17 | cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP), |
18 | asyncMiddleware(getSitemap) | 18 | asyncMiddleware(getSitemap) |
19 | ) | 19 | ) |
20 | 20 | ||
diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts index 435b12193..cdc6bfb8b 100644 --- a/server/controllers/feeds.ts +++ b/server/controllers/feeds.ts | |||
@@ -16,20 +16,20 @@ import { | |||
16 | videosSortValidator, | 16 | videosSortValidator, |
17 | videoSubscriptionFeedsValidator | 17 | videoSubscriptionFeedsValidator |
18 | } from '../middlewares' | 18 | } from '../middlewares' |
19 | import { cacheRoute } from '../middlewares/cache' | 19 | import { cacheRouteFactory } from '../middlewares/cache/cache' |
20 | import { VideoModel } from '../models/video/video' | 20 | import { VideoModel } from '../models/video/video' |
21 | import { VideoCommentModel } from '../models/video/video-comment' | 21 | import { VideoCommentModel } from '../models/video/video-comment' |
22 | 22 | ||
23 | const feedsRouter = express.Router() | 23 | const feedsRouter = express.Router() |
24 | 24 | ||
25 | const cacheRoute = cacheRouteFactory({ | ||
26 | headerBlacklist: [ 'Content-Type' ] | ||
27 | }) | ||
28 | |||
25 | feedsRouter.get('/feeds/video-comments.:format', | 29 | feedsRouter.get('/feeds/video-comments.:format', |
26 | feedsFormatValidator, | 30 | feedsFormatValidator, |
27 | setFeedFormatContentType, | 31 | setFeedFormatContentType, |
28 | asyncMiddleware(cacheRoute({ | 32 | cacheRoute(ROUTE_CACHE_LIFETIME.FEEDS), |
29 | headerBlacklist: [ | ||
30 | 'Content-Type' | ||
31 | ] | ||
32 | })(ROUTE_CACHE_LIFETIME.FEEDS)), | ||
33 | asyncMiddleware(videoFeedsValidator), | 33 | asyncMiddleware(videoFeedsValidator), |
34 | asyncMiddleware(videoCommentsFeedsValidator), | 34 | asyncMiddleware(videoCommentsFeedsValidator), |
35 | asyncMiddleware(generateVideoCommentsFeed) | 35 | asyncMiddleware(generateVideoCommentsFeed) |
@@ -40,11 +40,7 @@ feedsRouter.get('/feeds/videos.:format', | |||
40 | setDefaultVideosSort, | 40 | setDefaultVideosSort, |
41 | feedsFormatValidator, | 41 | feedsFormatValidator, |
42 | setFeedFormatContentType, | 42 | setFeedFormatContentType, |
43 | asyncMiddleware(cacheRoute({ | 43 | cacheRoute(ROUTE_CACHE_LIFETIME.FEEDS), |
44 | headerBlacklist: [ | ||
45 | 'Content-Type' | ||
46 | ] | ||
47 | })(ROUTE_CACHE_LIFETIME.FEEDS)), | ||
48 | commonVideosFiltersValidator, | 44 | commonVideosFiltersValidator, |
49 | asyncMiddleware(videoFeedsValidator), | 45 | asyncMiddleware(videoFeedsValidator), |
50 | asyncMiddleware(generateVideoFeed) | 46 | asyncMiddleware(generateVideoFeed) |
@@ -55,11 +51,7 @@ feedsRouter.get('/feeds/subscriptions.:format', | |||
55 | setDefaultVideosSort, | 51 | setDefaultVideosSort, |
56 | feedsFormatValidator, | 52 | feedsFormatValidator, |
57 | setFeedFormatContentType, | 53 | setFeedFormatContentType, |
58 | asyncMiddleware(cacheRoute({ | 54 | cacheRoute(ROUTE_CACHE_LIFETIME.FEEDS), |
59 | headerBlacklist: [ | ||
60 | 'Content-Type' | ||
61 | ] | ||
62 | })(ROUTE_CACHE_LIFETIME.FEEDS)), | ||
63 | commonVideosFiltersValidator, | 55 | commonVideosFiltersValidator, |
64 | asyncMiddleware(videoSubscriptionFeedsValidator), | 56 | asyncMiddleware(videoSubscriptionFeedsValidator), |
65 | asyncMiddleware(generateVideoFeedForSubscriptions) | 57 | asyncMiddleware(generateVideoFeedForSubscriptions) |
diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 5900eaff3..912d7e36c 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts | |||
@@ -19,7 +19,7 @@ import { | |||
19 | } from '../initializers/constants' | 19 | } from '../initializers/constants' |
20 | import { getThemeOrDefault } from '../lib/plugins/theme-utils' | 20 | import { getThemeOrDefault } from '../lib/plugins/theme-utils' |
21 | import { asyncMiddleware } from '../middlewares' | 21 | import { asyncMiddleware } from '../middlewares' |
22 | import { cacheRoute } from '../middlewares/cache' | 22 | import { cacheRoute } from '../middlewares/cache/cache' |
23 | import { UserModel } from '../models/user/user' | 23 | import { UserModel } from '../models/user/user' |
24 | import { VideoModel } from '../models/video/video' | 24 | import { VideoModel } from '../models/video/video' |
25 | import { VideoCommentModel } from '../models/video/video-comment' | 25 | import { VideoCommentModel } from '../models/video/video-comment' |
@@ -66,7 +66,7 @@ staticRouter.use( | |||
66 | 66 | ||
67 | // robots.txt service | 67 | // robots.txt service |
68 | staticRouter.get('/robots.txt', | 68 | staticRouter.get('/robots.txt', |
69 | asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.ROBOTS)), | 69 | cacheRoute(ROUTE_CACHE_LIFETIME.ROBOTS), |
70 | (_, res: express.Response) => { | 70 | (_, res: express.Response) => { |
71 | res.type('text/plain') | 71 | res.type('text/plain') |
72 | return res.send(CONFIG.INSTANCE.ROBOTS) | 72 | return res.send(CONFIG.INSTANCE.ROBOTS) |
@@ -86,7 +86,7 @@ staticRouter.get('/security.txt', | |||
86 | ) | 86 | ) |
87 | 87 | ||
88 | staticRouter.get('/.well-known/security.txt', | 88 | staticRouter.get('/.well-known/security.txt', |
89 | asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.SECURITYTXT)), | 89 | cacheRoute(ROUTE_CACHE_LIFETIME.SECURITYTXT), |
90 | (_, res: express.Response) => { | 90 | (_, res: express.Response) => { |
91 | res.type('text/plain') | 91 | res.type('text/plain') |
92 | return res.send(CONFIG.INSTANCE.SECURITYTXT + CONFIG.INSTANCE.SECURITYTXT_CONTACT) | 92 | return res.send(CONFIG.INSTANCE.SECURITYTXT + CONFIG.INSTANCE.SECURITYTXT_CONTACT) |
@@ -95,7 +95,7 @@ staticRouter.get('/.well-known/security.txt', | |||
95 | 95 | ||
96 | // nodeinfo service | 96 | // nodeinfo service |
97 | staticRouter.use('/.well-known/nodeinfo', | 97 | staticRouter.use('/.well-known/nodeinfo', |
98 | asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.NODEINFO)), | 98 | cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO), |
99 | (_, res: express.Response) => { | 99 | (_, res: express.Response) => { |
100 | return res.json({ | 100 | return res.json({ |
101 | links: [ | 101 | links: [ |
@@ -108,13 +108,13 @@ staticRouter.use('/.well-known/nodeinfo', | |||
108 | } | 108 | } |
109 | ) | 109 | ) |
110 | staticRouter.use('/nodeinfo/:version.json', | 110 | staticRouter.use('/nodeinfo/:version.json', |
111 | asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.NODEINFO)), | 111 | cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO), |
112 | asyncMiddleware(generateNodeinfo) | 112 | asyncMiddleware(generateNodeinfo) |
113 | ) | 113 | ) |
114 | 114 | ||
115 | // dnt-policy.txt service (see https://www.eff.org/dnt-policy) | 115 | // dnt-policy.txt service (see https://www.eff.org/dnt-policy) |
116 | staticRouter.use('/.well-known/dnt-policy.txt', | 116 | staticRouter.use('/.well-known/dnt-policy.txt', |
117 | asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.DNT_POLICY)), | 117 | cacheRoute(ROUTE_CACHE_LIFETIME.DNT_POLICY), |
118 | (_, res: express.Response) => { | 118 | (_, res: express.Response) => { |
119 | res.type('text/plain') | 119 | res.type('text/plain') |
120 | 120 | ||
diff --git a/server/middlewares/cache.ts b/server/middlewares/cache.ts deleted file mode 100644 index e508b22a6..000000000 --- a/server/middlewares/cache.ts +++ /dev/null | |||
@@ -1,28 +0,0 @@ | |||
1 | import * as apicache from 'apicache' | ||
2 | import { HttpStatusCode } from '../../shared/models/http/http-error-codes' | ||
3 | import { Redis } from '../lib/redis' | ||
4 | |||
5 | // Ensure Redis is initialized | ||
6 | Redis.Instance.init() | ||
7 | |||
8 | const defaultOptions = { | ||
9 | redisClient: Redis.Instance.getClient(), | ||
10 | appendKey: () => Redis.Instance.getPrefix(), | ||
11 | statusCodes: { | ||
12 | exclude: [ | ||
13 | HttpStatusCode.FORBIDDEN_403, | ||
14 | HttpStatusCode.NOT_FOUND_404 | ||
15 | ] | ||
16 | } | ||
17 | } | ||
18 | |||
19 | const cacheRoute = (extraOptions = {}) => apicache.options({ | ||
20 | ...defaultOptions, | ||
21 | ...extraOptions | ||
22 | }).middleware | ||
23 | |||
24 | // --------------------------------------------------------------------------- | ||
25 | |||
26 | export { | ||
27 | cacheRoute | ||
28 | } | ||
diff --git a/server/middlewares/cache/cache.ts b/server/middlewares/cache/cache.ts new file mode 100644 index 000000000..48162a0ae --- /dev/null +++ b/server/middlewares/cache/cache.ts | |||
@@ -0,0 +1,32 @@ | |||
1 | import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' | ||
2 | import { Redis } from '../../lib/redis' | ||
3 | import { ApiCache, APICacheOptions } from './shared' | ||
4 | |||
5 | // Ensure Redis is initialized | ||
6 | Redis.Instance.init() | ||
7 | |||
8 | const defaultOptions: APICacheOptions = { | ||
9 | excludeStatus: [ | ||
10 | HttpStatusCode.FORBIDDEN_403, | ||
11 | HttpStatusCode.NOT_FOUND_404 | ||
12 | ] | ||
13 | } | ||
14 | |||
15 | function cacheRoute (duration: string) { | ||
16 | const instance = new ApiCache(defaultOptions) | ||
17 | |||
18 | return instance.buildMiddleware(duration) | ||
19 | } | ||
20 | |||
21 | function cacheRouteFactory (options: APICacheOptions) { | ||
22 | const instance = new ApiCache({ ...defaultOptions, ...options }) | ||
23 | |||
24 | return instance.buildMiddleware.bind(instance) | ||
25 | } | ||
26 | |||
27 | // --------------------------------------------------------------------------- | ||
28 | |||
29 | export { | ||
30 | cacheRoute, | ||
31 | cacheRouteFactory | ||
32 | } | ||
diff --git a/server/middlewares/cache/index.ts b/server/middlewares/cache/index.ts new file mode 100644 index 000000000..79b512828 --- /dev/null +++ b/server/middlewares/cache/index.ts | |||
@@ -0,0 +1 @@ | |||
export * from './cache' | |||
diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts new file mode 100644 index 000000000..f9f7b1b67 --- /dev/null +++ b/server/middlewares/cache/shared/api-cache.ts | |||
@@ -0,0 +1,269 @@ | |||
1 | // Thanks: https://github.com/kwhitley/apicache | ||
2 | // We duplicated the library because it is unmaintened and prevent us to upgrade to recent NodeJS versions | ||
3 | |||
4 | import * as express from 'express' | ||
5 | import { OutgoingHttpHeaders } from 'http' | ||
6 | import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils' | ||
7 | import { logger } from '@server/helpers/logger' | ||
8 | import { Redis } from '@server/lib/redis' | ||
9 | import { HttpStatusCode } from '@shared/models' | ||
10 | |||
11 | export interface APICacheOptions { | ||
12 | headerBlacklist?: string[] | ||
13 | excludeStatus?: HttpStatusCode[] | ||
14 | } | ||
15 | |||
16 | interface CacheObject { | ||
17 | status: number | ||
18 | headers: OutgoingHttpHeaders | ||
19 | data: any | ||
20 | encoding: BufferEncoding | ||
21 | timestamp: number | ||
22 | } | ||
23 | |||
24 | export class ApiCache { | ||
25 | |||
26 | private readonly options: APICacheOptions | ||
27 | private readonly timers: { [ id: string ]: NodeJS.Timeout } = {} | ||
28 | |||
29 | private index: { all: string[] } = { all: [] } | ||
30 | |||
31 | constructor (options: APICacheOptions) { | ||
32 | this.options = { | ||
33 | headerBlacklist: [], | ||
34 | excludeStatus: [], | ||
35 | |||
36 | ...options | ||
37 | } | ||
38 | } | ||
39 | |||
40 | buildMiddleware (strDuration: string) { | ||
41 | const duration = parseDurationToMs(strDuration) | ||
42 | |||
43 | return (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
44 | const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl | ||
45 | const redis = Redis.Instance.getClient() | ||
46 | |||
47 | if (!redis.connected) return this.makeResponseCacheable(res, next, key, duration) | ||
48 | |||
49 | try { | ||
50 | redis.hgetall(key, (err, obj) => { | ||
51 | if (!err && obj && obj.response) { | ||
52 | return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration) | ||
53 | } | ||
54 | |||
55 | return this.makeResponseCacheable(res, next, key, duration) | ||
56 | }) | ||
57 | } catch (err) { | ||
58 | return this.makeResponseCacheable(res, next, key, duration) | ||
59 | } | ||
60 | } | ||
61 | } | ||
62 | |||
63 | private shouldCacheResponse (response: express.Response) { | ||
64 | if (!response) return false | ||
65 | if (this.options.excludeStatus.includes(response.statusCode)) return false | ||
66 | |||
67 | return true | ||
68 | } | ||
69 | |||
70 | private addIndexEntries (key: string) { | ||
71 | this.index.all.unshift(key) | ||
72 | } | ||
73 | |||
74 | private filterBlacklistedHeaders (headers: OutgoingHttpHeaders) { | ||
75 | return Object.keys(headers) | ||
76 | .filter(key => !this.options.headerBlacklist.includes(key)) | ||
77 | .reduce((acc, header) => { | ||
78 | acc[header] = headers[header] | ||
79 | |||
80 | return acc | ||
81 | }, {}) | ||
82 | } | ||
83 | |||
84 | private createCacheObject (status: number, headers: OutgoingHttpHeaders, data: any, encoding: BufferEncoding) { | ||
85 | return { | ||
86 | status, | ||
87 | headers: this.filterBlacklistedHeaders(headers), | ||
88 | data, | ||
89 | encoding, | ||
90 | |||
91 | // Seconds since epoch, used to properly decrement max-age headers in cached responses. | ||
92 | timestamp: new Date().getTime() / 1000 | ||
93 | } as CacheObject | ||
94 | } | ||
95 | |||
96 | private cacheResponse (key: string, value: object, duration: number) { | ||
97 | const redis = Redis.Instance.getClient() | ||
98 | |||
99 | if (redis.connected) { | ||
100 | try { | ||
101 | redis.hset(key, 'response', JSON.stringify(value)) | ||
102 | redis.hset(key, 'duration', duration + '') | ||
103 | redis.expire(key, duration / 1000) | ||
104 | } catch (err) { | ||
105 | logger.error('Cannot set cache in redis.', { err }) | ||
106 | } | ||
107 | } | ||
108 | |||
109 | // add automatic cache clearing from duration, includes max limit on setTimeout | ||
110 | this.timers[key] = setTimeout(() => this.clear(key), Math.min(duration, 2147483647)) | ||
111 | } | ||
112 | |||
113 | private accumulateContent (res: express.Response, content: any) { | ||
114 | if (!content) return | ||
115 | |||
116 | if (typeof content === 'string') { | ||
117 | res.locals.apicache.content = (res.locals.apicache.content || '') + content | ||
118 | return | ||
119 | } | ||
120 | |||
121 | if (Buffer.isBuffer(content)) { | ||
122 | let oldContent = res.locals.apicache.content | ||
123 | |||
124 | if (typeof oldContent === 'string') { | ||
125 | oldContent = Buffer.from(oldContent) | ||
126 | } | ||
127 | |||
128 | if (!oldContent) { | ||
129 | oldContent = Buffer.alloc(0) | ||
130 | } | ||
131 | |||
132 | res.locals.apicache.content = Buffer.concat( | ||
133 | [ oldContent, content ], | ||
134 | oldContent.length + content.length | ||
135 | ) | ||
136 | |||
137 | return | ||
138 | } | ||
139 | |||
140 | res.locals.apicache.content = content | ||
141 | } | ||
142 | |||
143 | private makeResponseCacheable (res: express.Response, next: express.NextFunction, key: string, duration: number) { | ||
144 | const self = this | ||
145 | |||
146 | res.locals.apicache = { | ||
147 | write: res.write, | ||
148 | writeHead: res.writeHead, | ||
149 | end: res.end, | ||
150 | cacheable: true, | ||
151 | content: undefined, | ||
152 | headers: {} | ||
153 | } | ||
154 | |||
155 | // Patch express | ||
156 | res.writeHead = function () { | ||
157 | if (self.shouldCacheResponse(res)) { | ||
158 | res.setHeader('cache-control', 'max-age=' + (duration / 1000).toFixed(0)) | ||
159 | } else { | ||
160 | res.setHeader('cache-control', 'no-cache, no-store, must-revalidate') | ||
161 | } | ||
162 | |||
163 | res.locals.apicache.headers = Object.assign({}, res.getHeaders()) | ||
164 | return res.locals.apicache.writeHead.apply(this, arguments as any) | ||
165 | } | ||
166 | |||
167 | res.write = function (chunk: any) { | ||
168 | self.accumulateContent(res, chunk) | ||
169 | return res.locals.apicache.write.apply(this, arguments as any) | ||
170 | } | ||
171 | |||
172 | res.end = function (content: any, encoding: BufferEncoding) { | ||
173 | if (self.shouldCacheResponse(res)) { | ||
174 | self.accumulateContent(res, content) | ||
175 | |||
176 | if (res.locals.apicache.cacheable && res.locals.apicache.content) { | ||
177 | self.addIndexEntries(key) | ||
178 | |||
179 | const headers = res.locals.apicache.headers || res.getHeaders() | ||
180 | const cacheObject = self.createCacheObject( | ||
181 | res.statusCode, | ||
182 | headers, | ||
183 | res.locals.apicache.content, | ||
184 | encoding | ||
185 | ) | ||
186 | self.cacheResponse(key, cacheObject, duration) | ||
187 | } | ||
188 | } | ||
189 | |||
190 | res.locals.apicache.end.apply(this, arguments as any) | ||
191 | } as any | ||
192 | |||
193 | next() | ||
194 | } | ||
195 | |||
196 | private sendCachedResponse (request: express.Request, response: express.Response, cacheObject: CacheObject, duration: number) { | ||
197 | const headers = response.getHeaders() | ||
198 | |||
199 | if (isTestInstance()) { | ||
200 | Object.assign(headers, { | ||
201 | 'x-api-cache-cached': 'true' | ||
202 | }) | ||
203 | } | ||
204 | |||
205 | Object.assign(headers, this.filterBlacklistedHeaders(cacheObject.headers || {}), { | ||
206 | // Set properly decremented max-age header | ||
207 | // This ensures that max-age is in sync with the cache expiration | ||
208 | 'cache-control': | ||
209 | 'max-age=' + | ||
210 | Math.max( | ||
211 | 0, | ||
212 | (duration / 1000 - (new Date().getTime() / 1000 - cacheObject.timestamp)) | ||
213 | ).toFixed(0) | ||
214 | }) | ||
215 | |||
216 | // unstringify buffers | ||
217 | let data = cacheObject.data | ||
218 | if (data && data.type === 'Buffer') { | ||
219 | data = typeof data.data === 'number' | ||
220 | ? Buffer.alloc(data.data) | ||
221 | : Buffer.from(data.data) | ||
222 | } | ||
223 | |||
224 | // Test Etag against If-None-Match for 304 | ||
225 | const cachedEtag = cacheObject.headers.etag | ||
226 | const requestEtag = request.headers['if-none-match'] | ||
227 | |||
228 | if (requestEtag && cachedEtag === requestEtag) { | ||
229 | response.writeHead(304, headers) | ||
230 | return response.end() | ||
231 | } | ||
232 | |||
233 | response.writeHead(cacheObject.status || 200, headers) | ||
234 | |||
235 | return response.end(data, cacheObject.encoding) | ||
236 | } | ||
237 | |||
238 | private clear (target: string) { | ||
239 | const redis = Redis.Instance.getClient() | ||
240 | |||
241 | if (target) { | ||
242 | clearTimeout(this.timers[target]) | ||
243 | delete this.timers[target] | ||
244 | |||
245 | try { | ||
246 | redis.del(target) | ||
247 | } catch (err) { | ||
248 | logger.error('Cannot delete %s in redis cache.', target, { err }) | ||
249 | } | ||
250 | |||
251 | this.index.all = this.index.all.filter(key => key !== target) | ||
252 | } else { | ||
253 | for (const key of this.index.all) { | ||
254 | clearTimeout(this.timers[key]) | ||
255 | delete this.timers[key] | ||
256 | |||
257 | try { | ||
258 | redis.del(key) | ||
259 | } catch (err) { | ||
260 | logger.error('Cannot delete %s in redis cache.', key, { err }) | ||
261 | } | ||
262 | } | ||
263 | |||
264 | this.index.all = [] | ||
265 | } | ||
266 | |||
267 | return this.index | ||
268 | } | ||
269 | } | ||
diff --git a/server/middlewares/cache/shared/index.ts b/server/middlewares/cache/shared/index.ts new file mode 100644 index 000000000..c707eaf7a --- /dev/null +++ b/server/middlewares/cache/shared/index.ts | |||
@@ -0,0 +1 @@ | |||
export * from './api-cache' | |||
diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts index 413653dac..a0035f623 100644 --- a/server/middlewares/index.ts +++ b/server/middlewares/index.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | export * from './validators' | 1 | export * from './validators' |
2 | export * from './cache' | ||
2 | export * from './activitypub' | 3 | export * from './activitypub' |
3 | export * from './async' | 4 | export * from './async' |
4 | export * from './auth' | 5 | export * from './auth' |
diff --git a/server/tests/feeds/feeds.ts b/server/tests/feeds/feeds.ts index 5667207c0..55b434846 100644 --- a/server/tests/feeds/feeds.ts +++ b/server/tests/feeds/feeds.ts | |||
@@ -8,6 +8,7 @@ import { | |||
8 | createMultipleServers, | 8 | createMultipleServers, |
9 | createSingleServer, | 9 | createSingleServer, |
10 | doubleFollow, | 10 | doubleFollow, |
11 | makeGetRequest, | ||
11 | PeerTubeServer, | 12 | PeerTubeServer, |
12 | setAccessTokensToServers, | 13 | setAccessTokensToServers, |
13 | waitJobs | 14 | waitJobs |
@@ -52,9 +53,7 @@ describe('Test syndication feeds', () => { | |||
52 | } | 53 | } |
53 | 54 | ||
54 | { | 55 | { |
55 | const attr = { username: 'john', password: 'password' } | 56 | userAccessToken = await servers[0].users.generateUserAndToken('john') |
56 | await servers[0].users.create({ username: attr.username, password: attr.password }) | ||
57 | userAccessToken = await servers[0].login.getAccessToken(attr) | ||
58 | 57 | ||
59 | const user = await servers[0].users.getMyInfo({ token: userAccessToken }) | 58 | const user = await servers[0].users.getMyInfo({ token: userAccessToken }) |
60 | userAccountId = user.account.id | 59 | userAccountId = user.account.id |
@@ -108,6 +107,41 @@ describe('Test syndication feeds', () => { | |||
108 | expect(JSON.parse(jsonText)).to.be.jsonSchema({ type: 'object' }) | 107 | expect(JSON.parse(jsonText)).to.be.jsonSchema({ type: 'object' }) |
109 | } | 108 | } |
110 | }) | 109 | }) |
110 | |||
111 | it('Should serve the endpoint with a classic request', async function () { | ||
112 | await makeGetRequest({ | ||
113 | url: servers[0].url, | ||
114 | path: '/feeds/videos.xml', | ||
115 | accept: 'application/xml', | ||
116 | expectedStatus: HttpStatusCode.OK_200 | ||
117 | }) | ||
118 | }) | ||
119 | |||
120 | it('Should serve the endpoint as a cached request', async function () { | ||
121 | const res = await makeGetRequest({ | ||
122 | url: servers[0].url, | ||
123 | path: '/feeds/videos.xml', | ||
124 | accept: 'application/xml', | ||
125 | expectedStatus: HttpStatusCode.OK_200 | ||
126 | }) | ||
127 | |||
128 | expect(res.headers['x-api-cache-cached']).to.equal('true') | ||
129 | }) | ||
130 | |||
131 | it('Should not serve the endpoint as a cached request', async function () { | ||
132 | const res = await makeGetRequest({ | ||
133 | url: servers[0].url, | ||
134 | path: '/feeds/videos.xml?v=186', | ||
135 | accept: 'application/xml', | ||
136 | expectedStatus: HttpStatusCode.OK_200 | ||
137 | }) | ||
138 | |||
139 | expect(res.headers['x-api-cache-cached']).to.not.exist | ||
140 | }) | ||
141 | |||
142 | it('Should refuse to serve the endpoint without accept header', async function () { | ||
143 | await makeGetRequest({ url: servers[0].url, path: '/feeds/videos.xml', expectedStatus: HttpStatusCode.NOT_ACCEPTABLE_406 }) | ||
144 | }) | ||
111 | }) | 145 | }) |
112 | 146 | ||
113 | describe('Videos feed', function () { | 147 | describe('Videos feed', function () { |
diff --git a/server/typings/express/index.d.ts b/server/typings/express/index.d.ts index 5469f3b83..1a99b598a 100644 --- a/server/typings/express/index.d.ts +++ b/server/typings/express/index.d.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | 1 | ||
2 | import { OutgoingHttpHeaders } from 'http' | ||
2 | import { RegisterServerAuthExternalOptions } from '@server/types' | 3 | import { RegisterServerAuthExternalOptions } from '@server/types' |
3 | import { | 4 | import { |
4 | MAbuseMessage, | 5 | MAbuseMessage, |
@@ -40,6 +41,7 @@ import { | |||
40 | MVideoShareActor, | 41 | MVideoShareActor, |
41 | MVideoThumbnail | 42 | MVideoThumbnail |
42 | } from '../../types/models' | 43 | } from '../../types/models' |
44 | import { Writable } from 'stream' | ||
43 | 45 | ||
44 | declare module 'express' { | 46 | declare module 'express' { |
45 | export interface Request { | 47 | export interface Request { |
@@ -98,6 +100,15 @@ declare module 'express' { | |||
98 | }) => void | 100 | }) => void |
99 | 101 | ||
100 | locals: { | 102 | locals: { |
103 | apicache: { | ||
104 | content: string | Buffer | ||
105 | write: Writable['write'] | ||
106 | writeHead: Response['writeHead'] | ||
107 | end: Response['end'] | ||
108 | cacheable: boolean | ||
109 | headers: OutgoingHttpHeaders | ||
110 | } | ||
111 | |||
101 | docUrl?: string | 112 | docUrl?: string |
102 | 113 | ||
103 | videoAPI?: MVideoFormattableDetails | 114 | videoAPI?: MVideoFormattableDetails |
@@ -1243,11 +1243,6 @@ anymatch@~3.1.1, anymatch@~3.1.2: | |||
1243 | normalize-path "^3.0.0" | 1243 | normalize-path "^3.0.0" |
1244 | picomatch "^2.0.4" | 1244 | picomatch "^2.0.4" |
1245 | 1245 | ||
1246 | apicache@1.6.2: | ||
1247 | version "1.6.2" | ||
1248 | resolved "https://registry.yarnpkg.com/apicache/-/apicache-1.6.2.tgz#a0a3d51024fa2814c4ace7e9e7053ebcb0920ee6" | ||
1249 | integrity sha512-3z5e+1E2qwZoqzFVgdx5l9nGhSG0kHv3v2G170vnJSz5uj4mCLVZfRw0o37aWwV8pTPXSkB8OBZz3TIur4H26g== | ||
1250 | |||
1251 | append-field@^1.0.0: | 1246 | append-field@^1.0.0: |
1252 | version "1.0.0" | 1247 | version "1.0.0" |
1253 | resolved "https://registry.yarnpkg.com/append-field/-/append-field-1.0.0.tgz#1e3440e915f0b1203d23748e78edd7b9b5b43e56" | 1248 | resolved "https://registry.yarnpkg.com/append-field/-/append-field-1.0.0.tgz#1e3440e915f0b1203d23748e78edd7b9b5b43e56" |