From e5d91a9b9cc27b8de55dcf299c8569c89e23debb Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 24 Dec 2021 14:49:03 +0100 Subject: Upgrade redis dep --- server/lib/plugins/plugin-manager.ts | 8 +- server/lib/redis.ts | 168 ++++++--------------------- server/middlewares/cache/shared/api-cache.ts | 50 ++++---- 3 files changed, 69 insertions(+), 157 deletions(-) (limited to 'server') diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts index ff00ab9e8..39e7f9a5b 100644 --- a/server/lib/plugins/plugin-manager.ts +++ b/server/lib/plugins/plugin-manager.ts @@ -5,7 +5,13 @@ import { basename, join } from 'path' import { decachePlugin } from '@server/helpers/decache' import { MOAuthTokenUser, MUser } from '@server/types/models' import { getCompleteLocale } from '@shared/core-utils' -import { ClientScriptJSON, PluginPackageJSON, PluginTranslation, PluginTranslationPathsJSON, RegisterServerHookOptions } from '@shared/models' +import { + ClientScriptJSON, + PluginPackageJSON, + PluginTranslation, + PluginTranslationPathsJSON, + RegisterServerHookOptions +} from '@shared/models' import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' import { PluginType } from '../../../shared/models/plugins/plugin.type' import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model' diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 8aec4b793..0478bfc89 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -1,31 +1,29 @@ import express from 'express' -import { createClient, RedisClient } from 'redis' +import { createClient } from 'redis' +import { exists } from '@server/helpers/custom-validators/misc' import { logger } from '../helpers/logger' import { generateRandomString } from '../helpers/utils' +import { CONFIG } from '../initializers/config' import { CONTACT_FORM_LIFETIME, + RESUMABLE_UPLOAD_SESSION_LIFETIME, + TRACKER_RATE_LIMITS, USER_EMAIL_VERIFY_LIFETIME, - USER_PASSWORD_RESET_LIFETIME, USER_PASSWORD_CREATE_LIFETIME, + USER_PASSWORD_RESET_LIFETIME, VIEW_LIFETIME, - WEBSERVER, - TRACKER_RATE_LIMITS, - RESUMABLE_UPLOAD_SESSION_LIFETIME + WEBSERVER } from '../initializers/constants' -import { CONFIG } from '../initializers/config' -import { exists } from '@server/helpers/custom-validators/misc' -type CachedRoute = { - body: string - contentType?: string - statusCode?: string -} +// Only used for typings +const redisClientWrapperForType = () => createClient<{}>() class Redis { private static instance: Redis private initialized = false - private client: RedisClient + private connected = false + private client: ReturnType private prefix: string private constructor () { @@ -38,21 +36,24 @@ class Redis { this.client = createClient(Redis.getRedisClientOptions()) + this.client.connect() + .then(() => { this.connected = true }) + .catch(err => { + logger.error('Cannot connect to redis', { err }) + process.exit(-1) + }) + this.client.on('error', err => { logger.error('Error in Redis client.', { err }) process.exit(-1) }) - if (CONFIG.REDIS.AUTH) { - this.client.auth(CONFIG.REDIS.AUTH) - } - this.prefix = 'redis-' + WEBSERVER.HOST + '-' } static getRedisClientOptions () { return Object.assign({}, - (CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {}, + CONFIG.REDIS.AUTH ? { password: CONFIG.REDIS.AUTH } : {}, (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {}, (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT) ? { host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT } @@ -68,6 +69,10 @@ class Redis { return this.prefix } + isConnected () { + return this.connected + } + /* ************ Forgot password ************ */ async setResetPasswordVerificationString (userId: number) { @@ -146,25 +151,6 @@ class Redis { return this.exists(this.generateTrackerBlockIPKey(ip)) } - /* ************ API cache ************ */ - - async getCachedRoute (req: express.Request) { - const cached = await this.getObject(this.generateCachedRouteKey(req)) - - return cached as CachedRoute - } - - setCachedRoute (req: express.Request, body: any, lifetime: number, contentType?: string, statusCode?: number) { - const cached: CachedRoute = Object.assign( - {}, - { body: body.toString() }, - (contentType) ? { contentType } : null, - (statusCode) ? { statusCode: statusCode.toString() } : null - ) - - return this.setObject(this.generateCachedRouteKey(req), cached, lifetime) - } - /* ************ Video views stats ************ */ addVideoViewStats (videoId: number) { @@ -277,10 +263,6 @@ class Redis { /* ************ Keys generation ************ */ - generateCachedRouteKey (req: express.Request) { - return req.method + '-' + req.originalUrl - } - private generateLocalVideoViewsKeys (videoId?: Number) { return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } } @@ -320,125 +302,45 @@ class Redis { /* ************ Redis helpers ************ */ private getValue (key: string) { - return new Promise((res, rej) => { - this.client.get(this.prefix + key, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) + return this.client.get(this.prefix + key) } private getSet (key: string) { - return new Promise((res, rej) => { - this.client.smembers(this.prefix + key, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) + return this.client.sMembers(this.prefix + key) } private addToSet (key: string, value: string) { - return new Promise((res, rej) => { - this.client.sadd(this.prefix + key, value, err => err ? rej(err) : res()) - }) + return this.client.sAdd(this.prefix + key, value) } private deleteFromSet (key: string, value: string) { - return new Promise((res, rej) => { - this.client.srem(this.prefix + key, value, err => err ? rej(err) : res()) - }) + return this.client.sRem(this.prefix + key, value) } private deleteKey (key: string) { - return new Promise((res, rej) => { - this.client.del(this.prefix + key, err => err ? rej(err) : res()) - }) - } - - private deleteFieldInHash (key: string, field: string) { - return new Promise((res, rej) => { - this.client.hdel(this.prefix + key, field, err => err ? rej(err) : res()) - }) + return this.client.del(this.prefix + key) } - private setValue (key: string, value: string, expirationMilliseconds: number) { - return new Promise((res, rej) => { - this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => { - if (err) return rej(err) - - if (ok !== 'OK') return rej(new Error('Redis set result is not OK.')) + private async setValue (key: string, value: string, expirationMilliseconds: number) { + const result = await this.client.set(this.prefix + key, value, { PX: expirationMilliseconds }) - return res() - }) - }) + if (result !== 'OK') throw new Error('Redis set result is not OK.') } private removeValue (key: string) { - return new Promise((res, rej) => { - this.client.del(this.prefix + key, err => { - if (err) return rej(err) - - return res() - }) - }) - } - - private setObject (key: string, obj: { [id: string]: string }, expirationMilliseconds: number) { - return new Promise((res, rej) => { - this.client.hmset(this.prefix + key, obj, (err, ok) => { - if (err) return rej(err) - if (!ok) return rej(new Error('Redis mset result is not OK.')) - - this.client.pexpire(this.prefix + key, expirationMilliseconds, (err, ok) => { - if (err) return rej(err) - if (!ok) return rej(new Error('Redis expiration result is not OK.')) - - return res() - }) - }) - }) + return this.client.del(this.prefix + key) } private getObject (key: string) { - return new Promise<{ [id: string]: string }>((res, rej) => { - this.client.hgetall(this.prefix + key, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) - } - - private setValueInHash (key: string, field: string, value: string) { - return new Promise((res, rej) => { - this.client.hset(this.prefix + key, field, value, (err) => { - if (err) return rej(err) - - return res() - }) - }) + return this.client.hGetAll(this.prefix + key) } private increment (key: string) { - return new Promise((res, rej) => { - this.client.incr(this.prefix + key, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) + return this.client.incr(this.prefix + key) } private exists (key: string) { - return new Promise((res, rej) => { - this.client.exists(this.prefix + key, (err, existsNumber) => { - if (err) return rej(err) - - return res(existsNumber === 1) - }) - }) + return this.client.exists(this.prefix + key) } static get Instance () { diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts index f8846dcfc..86c5095b5 100644 --- a/server/middlewares/cache/shared/api-cache.ts +++ b/server/middlewares/cache/shared/api-cache.ts @@ -7,6 +7,7 @@ import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils' import { logger } from '@server/helpers/logger' import { Redis } from '@server/lib/redis' import { HttpStatusCode } from '@shared/models' +import { asyncMiddleware } from '@server/middlewares' export interface APICacheOptions { headerBlacklist?: string[] @@ -40,24 +41,25 @@ export class ApiCache { buildMiddleware (strDuration: string) { const duration = parseDurationToMs(strDuration) - return (req: express.Request, res: express.Response, next: express.NextFunction) => { - const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl - const redis = Redis.Instance.getClient() + return asyncMiddleware( + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl + const redis = Redis.Instance.getClient() - if (!redis.connected) return this.makeResponseCacheable(res, next, key, duration) + if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration) - try { - redis.hgetall(key, (err, obj) => { - if (!err && obj && obj.response) { + try { + const obj = await redis.hGetAll(key) + if (obj?.response) { return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration) } return this.makeResponseCacheable(res, next, key, duration) - }) - } catch (err) { - return this.makeResponseCacheable(res, next, key, duration) + } catch (err) { + return this.makeResponseCacheable(res, next, key, duration) + } } - } + ) } private shouldCacheResponse (response: express.Response) { @@ -93,21 +95,22 @@ export class ApiCache { } as CacheObject } - private cacheResponse (key: string, value: object, duration: number) { + private async cacheResponse (key: string, value: object, duration: number) { const redis = Redis.Instance.getClient() - if (redis.connected) { - try { - redis.hset(key, 'response', JSON.stringify(value)) - redis.hset(key, 'duration', duration + '') + if (Redis.Instance.isConnected()) { + await Promise.all([ + redis.hSet(key, 'response', JSON.stringify(value)), + redis.hSet(key, 'duration', duration + ''), redis.expire(key, duration / 1000) - } catch (err) { - logger.error('Cannot set cache in redis.', { err }) - } + ]) } // add automatic cache clearing from duration, includes max limit on setTimeout - this.timers[key] = setTimeout(() => this.clear(key), Math.min(duration, 2147483647)) + this.timers[key] = setTimeout(() => { + this.clear(key) + .catch(err => logger.error('Cannot clear Redis key %s.', key, { err })) + }, Math.min(duration, 2147483647)) } private accumulateContent (res: express.Response, content: any) { @@ -184,6 +187,7 @@ export class ApiCache { encoding ) self.cacheResponse(key, cacheObject, duration) + .catch(err => logger.error('Cannot cache response', { err })) } } @@ -235,7 +239,7 @@ export class ApiCache { return response.end(data, cacheObject.encoding) } - private clear (target: string) { + private async clear (target: string) { const redis = Redis.Instance.getClient() if (target) { @@ -243,7 +247,7 @@ export class ApiCache { delete this.timers[target] try { - redis.del(target) + await redis.del(target) } catch (err) { logger.error('Cannot delete %s in redis cache.', target, { err }) } @@ -255,7 +259,7 @@ export class ApiCache { delete this.timers[key] try { - redis.del(key) + await redis.del(key) } catch (err) { logger.error('Cannot delete %s in redis cache.', key, { err }) } -- cgit v1.2.3