aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-12-24 14:49:03 +0100
committerChocobozzz <me@florianbigard.com>2021-12-24 14:52:24 +0100
commite5d91a9b9cc27b8de55dcf299c8569c89e23debb (patch)
treeafa8c9908ddd3e02c0f9a81b81cf61a4ea1469e5 /server
parent2d28b0c21dc066105677374419acd7ba8e919e8d (diff)
downloadPeerTube-e5d91a9b9cc27b8de55dcf299c8569c89e23debb.tar.gz
PeerTube-e5d91a9b9cc27b8de55dcf299c8569c89e23debb.tar.zst
PeerTube-e5d91a9b9cc27b8de55dcf299c8569c89e23debb.zip
Upgrade redis dep
Diffstat (limited to 'server')
-rw-r--r--server/lib/plugins/plugin-manager.ts8
-rw-r--r--server/lib/redis.ts168
-rw-r--r--server/middlewares/cache/shared/api-cache.ts50
3 files changed, 69 insertions, 157 deletions
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts
index ff00ab9e8..39e7f9a5b 100644
--- a/server/lib/plugins/plugin-manager.ts
+++ b/server/lib/plugins/plugin-manager.ts
@@ -5,7 +5,13 @@ import { basename, join } from 'path'
5import { decachePlugin } from '@server/helpers/decache' 5import { decachePlugin } from '@server/helpers/decache'
6import { MOAuthTokenUser, MUser } from '@server/types/models' 6import { MOAuthTokenUser, MUser } from '@server/types/models'
7import { getCompleteLocale } from '@shared/core-utils' 7import { getCompleteLocale } from '@shared/core-utils'
8import { ClientScriptJSON, PluginPackageJSON, PluginTranslation, PluginTranslationPathsJSON, RegisterServerHookOptions } from '@shared/models' 8import {
9 ClientScriptJSON,
10 PluginPackageJSON,
11 PluginTranslation,
12 PluginTranslationPathsJSON,
13 RegisterServerHookOptions
14} from '@shared/models'
9import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' 15import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks'
10import { PluginType } from '../../../shared/models/plugins/plugin.type' 16import { PluginType } from '../../../shared/models/plugins/plugin.type'
11import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model' 17import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model'
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 8aec4b793..0478bfc89 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -1,31 +1,29 @@
1import express from 'express' 1import express from 'express'
2import { createClient, RedisClient } from 'redis' 2import { createClient } from 'redis'
3import { exists } from '@server/helpers/custom-validators/misc'
3import { logger } from '../helpers/logger' 4import { logger } from '../helpers/logger'
4import { generateRandomString } from '../helpers/utils' 5import { generateRandomString } from '../helpers/utils'
6import { CONFIG } from '../initializers/config'
5import { 7import {
6 CONTACT_FORM_LIFETIME, 8 CONTACT_FORM_LIFETIME,
9 RESUMABLE_UPLOAD_SESSION_LIFETIME,
10 TRACKER_RATE_LIMITS,
7 USER_EMAIL_VERIFY_LIFETIME, 11 USER_EMAIL_VERIFY_LIFETIME,
8 USER_PASSWORD_RESET_LIFETIME,
9 USER_PASSWORD_CREATE_LIFETIME, 12 USER_PASSWORD_CREATE_LIFETIME,
13 USER_PASSWORD_RESET_LIFETIME,
10 VIEW_LIFETIME, 14 VIEW_LIFETIME,
11 WEBSERVER, 15 WEBSERVER
12 TRACKER_RATE_LIMITS,
13 RESUMABLE_UPLOAD_SESSION_LIFETIME
14} from '../initializers/constants' 16} from '../initializers/constants'
15import { CONFIG } from '../initializers/config'
16import { exists } from '@server/helpers/custom-validators/misc'
17 17
18type CachedRoute = { 18// Only used for typings
19 body: string 19const redisClientWrapperForType = () => createClient<{}>()
20 contentType?: string
21 statusCode?: string
22}
23 20
24class Redis { 21class Redis {
25 22
26 private static instance: Redis 23 private static instance: Redis
27 private initialized = false 24 private initialized = false
28 private client: RedisClient 25 private connected = false
26 private client: ReturnType<typeof redisClientWrapperForType>
29 private prefix: string 27 private prefix: string
30 28
31 private constructor () { 29 private constructor () {
@@ -38,21 +36,24 @@ class Redis {
38 36
39 this.client = createClient(Redis.getRedisClientOptions()) 37 this.client = createClient(Redis.getRedisClientOptions())
40 38
39 this.client.connect()
40 .then(() => { this.connected = true })
41 .catch(err => {
42 logger.error('Cannot connect to redis', { err })
43 process.exit(-1)
44 })
45
41 this.client.on('error', err => { 46 this.client.on('error', err => {
42 logger.error('Error in Redis client.', { err }) 47 logger.error('Error in Redis client.', { err })
43 process.exit(-1) 48 process.exit(-1)
44 }) 49 })
45 50
46 if (CONFIG.REDIS.AUTH) {
47 this.client.auth(CONFIG.REDIS.AUTH)
48 }
49
50 this.prefix = 'redis-' + WEBSERVER.HOST + '-' 51 this.prefix = 'redis-' + WEBSERVER.HOST + '-'
51 } 52 }
52 53
53 static getRedisClientOptions () { 54 static getRedisClientOptions () {
54 return Object.assign({}, 55 return Object.assign({},
55 (CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {}, 56 CONFIG.REDIS.AUTH ? { password: CONFIG.REDIS.AUTH } : {},
56 (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {}, 57 (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {},
57 (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT) 58 (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT)
58 ? { host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT } 59 ? { host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT }
@@ -68,6 +69,10 @@ class Redis {
68 return this.prefix 69 return this.prefix
69 } 70 }
70 71
72 isConnected () {
73 return this.connected
74 }
75
71 /* ************ Forgot password ************ */ 76 /* ************ Forgot password ************ */
72 77
73 async setResetPasswordVerificationString (userId: number) { 78 async setResetPasswordVerificationString (userId: number) {
@@ -146,25 +151,6 @@ class Redis {
146 return this.exists(this.generateTrackerBlockIPKey(ip)) 151 return this.exists(this.generateTrackerBlockIPKey(ip))
147 } 152 }
148 153
149 /* ************ API cache ************ */
150
151 async getCachedRoute (req: express.Request) {
152 const cached = await this.getObject(this.generateCachedRouteKey(req))
153
154 return cached as CachedRoute
155 }
156
157 setCachedRoute (req: express.Request, body: any, lifetime: number, contentType?: string, statusCode?: number) {
158 const cached: CachedRoute = Object.assign(
159 {},
160 { body: body.toString() },
161 (contentType) ? { contentType } : null,
162 (statusCode) ? { statusCode: statusCode.toString() } : null
163 )
164
165 return this.setObject(this.generateCachedRouteKey(req), cached, lifetime)
166 }
167
168 /* ************ Video views stats ************ */ 154 /* ************ Video views stats ************ */
169 155
170 addVideoViewStats (videoId: number) { 156 addVideoViewStats (videoId: number) {
@@ -277,10 +263,6 @@ class Redis {
277 263
278 /* ************ Keys generation ************ */ 264 /* ************ Keys generation ************ */
279 265
280 generateCachedRouteKey (req: express.Request) {
281 return req.method + '-' + req.originalUrl
282 }
283
284 private generateLocalVideoViewsKeys (videoId?: Number) { 266 private generateLocalVideoViewsKeys (videoId?: Number) {
285 return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } 267 return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` }
286 } 268 }
@@ -320,125 +302,45 @@ class Redis {
320 /* ************ Redis helpers ************ */ 302 /* ************ Redis helpers ************ */
321 303
322 private getValue (key: string) { 304 private getValue (key: string) {
323 return new Promise<string>((res, rej) => { 305 return this.client.get(this.prefix + key)
324 this.client.get(this.prefix + key, (err, value) => {
325 if (err) return rej(err)
326
327 return res(value)
328 })
329 })
330 } 306 }
331 307
332 private getSet (key: string) { 308 private getSet (key: string) {
333 return new Promise<string[]>((res, rej) => { 309 return this.client.sMembers(this.prefix + key)
334 this.client.smembers(this.prefix + key, (err, value) => {
335 if (err) return rej(err)
336
337 return res(value)
338 })
339 })
340 } 310 }
341 311
342 private addToSet (key: string, value: string) { 312 private addToSet (key: string, value: string) {
343 return new Promise<void>((res, rej) => { 313 return this.client.sAdd(this.prefix + key, value)
344 this.client.sadd(this.prefix + key, value, err => err ? rej(err) : res())
345 })
346 } 314 }
347 315
348 private deleteFromSet (key: string, value: string) { 316 private deleteFromSet (key: string, value: string) {
349 return new Promise<void>((res, rej) => { 317 return this.client.sRem(this.prefix + key, value)
350 this.client.srem(this.prefix + key, value, err => err ? rej(err) : res())
351 })
352 } 318 }
353 319
354 private deleteKey (key: string) { 320 private deleteKey (key: string) {
355 return new Promise<void>((res, rej) => { 321 return this.client.del(this.prefix + key)
356 this.client.del(this.prefix + key, err => err ? rej(err) : res())
357 })
358 }
359
360 private deleteFieldInHash (key: string, field: string) {
361 return new Promise<void>((res, rej) => {
362 this.client.hdel(this.prefix + key, field, err => err ? rej(err) : res())
363 })
364 } 322 }
365 323
366 private setValue (key: string, value: string, expirationMilliseconds: number) { 324 private async setValue (key: string, value: string, expirationMilliseconds: number) {
367 return new Promise<void>((res, rej) => { 325 const result = await this.client.set(this.prefix + key, value, { PX: expirationMilliseconds })
368 this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => {
369 if (err) return rej(err)
370
371 if (ok !== 'OK') return rej(new Error('Redis set result is not OK.'))
372 326
373 return res() 327 if (result !== 'OK') throw new Error('Redis set result is not OK.')
374 })
375 })
376 } 328 }
377 329
378 private removeValue (key: string) { 330 private removeValue (key: string) {
379 return new Promise<void>((res, rej) => { 331 return this.client.del(this.prefix + key)
380 this.client.del(this.prefix + key, err => {
381 if (err) return rej(err)
382
383 return res()
384 })
385 })
386 }
387
388 private setObject (key: string, obj: { [id: string]: string }, expirationMilliseconds: number) {
389 return new Promise<void>((res, rej) => {
390 this.client.hmset(this.prefix + key, obj, (err, ok) => {
391 if (err) return rej(err)
392 if (!ok) return rej(new Error('Redis mset result is not OK.'))
393
394 this.client.pexpire(this.prefix + key, expirationMilliseconds, (err, ok) => {
395 if (err) return rej(err)
396 if (!ok) return rej(new Error('Redis expiration result is not OK.'))
397
398 return res()
399 })
400 })
401 })
402 } 332 }
403 333
404 private getObject (key: string) { 334 private getObject (key: string) {
405 return new Promise<{ [id: string]: string }>((res, rej) => { 335 return this.client.hGetAll(this.prefix + key)
406 this.client.hgetall(this.prefix + key, (err, value) => {
407 if (err) return rej(err)
408
409 return res(value)
410 })
411 })
412 }
413
414 private setValueInHash (key: string, field: string, value: string) {
415 return new Promise<void>((res, rej) => {
416 this.client.hset(this.prefix + key, field, value, (err) => {
417 if (err) return rej(err)
418
419 return res()
420 })
421 })
422 } 336 }
423 337
424 private increment (key: string) { 338 private increment (key: string) {
425 return new Promise<number>((res, rej) => { 339 return this.client.incr(this.prefix + key)
426 this.client.incr(this.prefix + key, (err, value) => {
427 if (err) return rej(err)
428
429 return res(value)
430 })
431 })
432 } 340 }
433 341
434 private exists (key: string) { 342 private exists (key: string) {
435 return new Promise<boolean>((res, rej) => { 343 return this.client.exists(this.prefix + key)
436 this.client.exists(this.prefix + key, (err, existsNumber) => {
437 if (err) return rej(err)
438
439 return res(existsNumber === 1)
440 })
441 })
442 } 344 }
443 345
444 static get Instance () { 346 static get Instance () {
diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts
index f8846dcfc..86c5095b5 100644
--- a/server/middlewares/cache/shared/api-cache.ts
+++ b/server/middlewares/cache/shared/api-cache.ts
@@ -7,6 +7,7 @@ import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils'
7import { logger } from '@server/helpers/logger' 7import { logger } from '@server/helpers/logger'
8import { Redis } from '@server/lib/redis' 8import { Redis } from '@server/lib/redis'
9import { HttpStatusCode } from '@shared/models' 9import { HttpStatusCode } from '@shared/models'
10import { asyncMiddleware } from '@server/middlewares'
10 11
11export interface APICacheOptions { 12export interface APICacheOptions {
12 headerBlacklist?: string[] 13 headerBlacklist?: string[]
@@ -40,24 +41,25 @@ export class ApiCache {
40 buildMiddleware (strDuration: string) { 41 buildMiddleware (strDuration: string) {
41 const duration = parseDurationToMs(strDuration) 42 const duration = parseDurationToMs(strDuration)
42 43
43 return (req: express.Request, res: express.Response, next: express.NextFunction) => { 44 return asyncMiddleware(
44 const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl 45 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
45 const redis = Redis.Instance.getClient() 46 const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl
47 const redis = Redis.Instance.getClient()
46 48
47 if (!redis.connected) return this.makeResponseCacheable(res, next, key, duration) 49 if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration)
48 50
49 try { 51 try {
50 redis.hgetall(key, (err, obj) => { 52 const obj = await redis.hGetAll(key)
51 if (!err && obj && obj.response) { 53 if (obj?.response) {
52 return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration) 54 return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration)
53 } 55 }
54 56
55 return this.makeResponseCacheable(res, next, key, duration) 57 return this.makeResponseCacheable(res, next, key, duration)
56 }) 58 } catch (err) {
57 } catch (err) { 59 return this.makeResponseCacheable(res, next, key, duration)
58 return this.makeResponseCacheable(res, next, key, duration) 60 }
59 } 61 }
60 } 62 )
61 } 63 }
62 64
63 private shouldCacheResponse (response: express.Response) { 65 private shouldCacheResponse (response: express.Response) {
@@ -93,21 +95,22 @@ export class ApiCache {
93 } as CacheObject 95 } as CacheObject
94 } 96 }
95 97
96 private cacheResponse (key: string, value: object, duration: number) { 98 private async cacheResponse (key: string, value: object, duration: number) {
97 const redis = Redis.Instance.getClient() 99 const redis = Redis.Instance.getClient()
98 100
99 if (redis.connected) { 101 if (Redis.Instance.isConnected()) {
100 try { 102 await Promise.all([
101 redis.hset(key, 'response', JSON.stringify(value)) 103 redis.hSet(key, 'response', JSON.stringify(value)),
102 redis.hset(key, 'duration', duration + '') 104 redis.hSet(key, 'duration', duration + ''),
103 redis.expire(key, duration / 1000) 105 redis.expire(key, duration / 1000)
104 } catch (err) { 106 ])
105 logger.error('Cannot set cache in redis.', { err })
106 }
107 } 107 }
108 108
109 // add automatic cache clearing from duration, includes max limit on setTimeout 109 // add automatic cache clearing from duration, includes max limit on setTimeout
110 this.timers[key] = setTimeout(() => this.clear(key), Math.min(duration, 2147483647)) 110 this.timers[key] = setTimeout(() => {
111 this.clear(key)
112 .catch(err => logger.error('Cannot clear Redis key %s.', key, { err }))
113 }, Math.min(duration, 2147483647))
111 } 114 }
112 115
113 private accumulateContent (res: express.Response, content: any) { 116 private accumulateContent (res: express.Response, content: any) {
@@ -184,6 +187,7 @@ export class ApiCache {
184 encoding 187 encoding
185 ) 188 )
186 self.cacheResponse(key, cacheObject, duration) 189 self.cacheResponse(key, cacheObject, duration)
190 .catch(err => logger.error('Cannot cache response', { err }))
187 } 191 }
188 } 192 }
189 193
@@ -235,7 +239,7 @@ export class ApiCache {
235 return response.end(data, cacheObject.encoding) 239 return response.end(data, cacheObject.encoding)
236 } 240 }
237 241
238 private clear (target: string) { 242 private async clear (target: string) {
239 const redis = Redis.Instance.getClient() 243 const redis = Redis.Instance.getClient()
240 244
241 if (target) { 245 if (target) {
@@ -243,7 +247,7 @@ export class ApiCache {
243 delete this.timers[target] 247 delete this.timers[target]
244 248
245 try { 249 try {
246 redis.del(target) 250 await redis.del(target)
247 } catch (err) { 251 } catch (err) {
248 logger.error('Cannot delete %s in redis cache.', target, { err }) 252 logger.error('Cannot delete %s in redis cache.', target, { err })
249 } 253 }
@@ -255,7 +259,7 @@ export class ApiCache {
255 delete this.timers[key] 259 delete this.timers[key]
256 260
257 try { 261 try {
258 redis.del(key) 262 await redis.del(key)
259 } catch (err) { 263 } catch (err) {
260 logger.error('Cannot delete %s in redis cache.', key, { err }) 264 logger.error('Cannot delete %s in redis cache.', key, { err })
261 } 265 }