server/lib/redis.ts

   1 import IoRedis, { RedisOptions } from 'ioredis'
   2 import { exists } from '@server/helpers/custom-validators/misc'
   3 import { sha256 } from '@shared/extra-utils'
   4 import { logger } from '../helpers/logger'
   5 import { generateRandomString } from '../helpers/utils'
   6 import { CONFIG } from '../initializers/config'
   7 import {
   8   AP_CLEANER,
   9   CONTACT_FORM_LIFETIME,
  10   RESUMABLE_UPLOAD_SESSION_LIFETIME,
  11   TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
  12   EMAIL_VERIFY_LIFETIME,
  13   USER_PASSWORD_CREATE_LIFETIME,
  14   USER_PASSWORD_RESET_LIFETIME,
  15   VIEW_LIFETIME,
  16   WEBSERVER
  17 } from '../initializers/constants'
  18
  19 class Redis {
  20
  21   private static instance: Redis
  22   private initialized = false
  23   private connected = false
  24   private client: IoRedis
  25   private prefix: string
  26
  27   private constructor () {
  28   }
  29
  30   init () {
  31     // Already initialized
  32     if (this.initialized === true) return
  33     this.initialized = true
  34
  35     const redisMode = CONFIG.REDIS.SENTINEL.ENABLED ? 'sentinel' : 'standalone'
  36     logger.info('Connecting to redis ' + redisMode + '...')
  37
  38     this.client = new IoRedis(Redis.getRedisClientOptions('', { enableAutoPipelining: true }))
  39     this.client.on('error', err => logger.error('Redis failed to connect', { err }))
  40     this.client.on('connect', () => {
  41       logger.info('Connected to redis.')
  42
  43       this.connected = true
  44     })
  45     this.client.on('reconnecting', (ms) => {
  46       logger.error(`Reconnecting to redis in ${ms}.`)
  47     })
  48     this.client.on('close', () => {
  49       logger.error('Connection to redis has closed.')
  50       this.connected = false
  51     })
  52
  53     this.client.on('end', () => {
  54       logger.error('Connection to redis has closed and no more reconnects will be done.')
  55     })
  56
  57     this.prefix = 'redis-' + WEBSERVER.HOST + '-'
  58   }
  59
  60   static getRedisClientOptions (name?: string, options: RedisOptions = {}): RedisOptions {
  61     const connectionName = [ 'PeerTube', name ].join('')
  62     const connectTimeout = 20000 // Could be slow since node use sync call to compile PeerTube
  63
  64     if (CONFIG.REDIS.SENTINEL.ENABLED) {
  65       return {
  66         connectionName,
  67         connectTimeout,
  68         enableTLSForSentinelMode: CONFIG.REDIS.SENTINEL.ENABLE_TLS,
  69         sentinelPassword: CONFIG.REDIS.AUTH,
  70         sentinels: CONFIG.REDIS.SENTINEL.SENTINELS,
  71         name: CONFIG.REDIS.SENTINEL.MASTER_NAME,
  72         ...options
  73       }
  74     }
  75
  76     return {
  77       connectionName,
  78       connectTimeout,
  79       password: CONFIG.REDIS.AUTH,
  80       db: CONFIG.REDIS.DB,
  81       host: CONFIG.REDIS.HOSTNAME,
  82       port: CONFIG.REDIS.PORT,
  83       path: CONFIG.REDIS.SOCKET,
  84       showFriendlyErrorStack: true,
  85       ...options
  86     }
  87   }
  88
  89   getClient () {
  90     return this.client
  91   }
  92
  93   getPrefix () {
  94     return this.prefix
  95   }
  96
  97   isConnected () {
  98     return this.connected
  99   }
 100
 101   /* ************ Forgot password ************ */
 102
 103   async setResetPasswordVerificationString (userId: number) {
 104     const generatedString = await generateRandomString(32)
 105
 106     await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_RESET_LIFETIME)
 107
 108     return generatedString
 109   }
 110
 111   async setCreatePasswordVerificationString (userId: number) {
 112     const generatedString = await generateRandomString(32)
 113
 114     await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_CREATE_LIFETIME)
 115
 116     return generatedString
 117   }
 118
 119   async removePasswordVerificationString (userId: number) {
 120     return this.removeValue(this.generateResetPasswordKey(userId))
 121   }
 122
 123   async getResetPasswordVerificationString (userId: number) {
 124     return this.getValue(this.generateResetPasswordKey(userId))
 125   }
 126
 127   /* ************ Two factor auth request ************ */
 128
 129   async setTwoFactorRequest (userId: number, otpSecret: string) {
 130     const requestToken = await generateRandomString(32)
 131
 132     await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME)
 133
 134     return requestToken
 135   }
 136
 137   async getTwoFactorRequestToken (userId: number, requestToken: string) {
 138     return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken))
 139   }
 140
 141   /* ************ Email verification ************ */
 142
 143   async setUserVerifyEmailVerificationString (userId: number) {
 144     const generatedString = await generateRandomString(32)
 145
 146     await this.setValue(this.generateUserVerifyEmailKey(userId), generatedString, EMAIL_VERIFY_LIFETIME)
 147
 148     return generatedString
 149   }
 150
 151   async getUserVerifyEmailLink (userId: number) {
 152     return this.getValue(this.generateUserVerifyEmailKey(userId))
 153   }
 154
 155   async setRegistrationVerifyEmailVerificationString (registrationId: number) {
 156     const generatedString = await generateRandomString(32)
 157
 158     await this.setValue(this.generateRegistrationVerifyEmailKey(registrationId), generatedString, EMAIL_VERIFY_LIFETIME)
 159
 160     return generatedString
 161   }
 162
 163   async getRegistrationVerifyEmailLink (registrationId: number) {
 164     return this.getValue(this.generateRegistrationVerifyEmailKey(registrationId))
 165   }
 166
 167   /* ************ Contact form per IP ************ */
 168
 169   async setContactFormIp (ip: string) {
 170     return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME)
 171   }
 172
 173   async doesContactFormIpExist (ip: string) {
 174     return this.exists(this.generateContactFormKey(ip))
 175   }
 176
 177   /* ************ Views per IP ************ */
 178
 179   setIPVideoView (ip: string, videoUUID: string) {
 180     return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW)
 181   }
 182
 183   async doesVideoIPViewExist (ip: string, videoUUID: string) {
 184     return this.exists(this.generateIPViewKey(ip, videoUUID))
 185   }
 186
 187   /* ************ Video views stats ************ */
 188
 189   addVideoViewStats (videoId: number) {
 190     const { videoKey, setKey } = this.generateVideoViewStatsKeys({ videoId })
 191
 192     return Promise.all([
 193       this.addToSet(setKey, videoId.toString()),
 194       this.increment(videoKey)
 195     ])
 196   }
 197
 198   async getVideoViewsStats (videoId: number, hour: number) {
 199     const { videoKey } = this.generateVideoViewStatsKeys({ videoId, hour })
 200
 201     const valueString = await this.getValue(videoKey)
 202     const valueInt = parseInt(valueString, 10)
 203
 204     if (isNaN(valueInt)) {
 205       logger.error('Cannot get videos views stats of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString)
 206       return undefined
 207     }
 208
 209     return valueInt
 210   }
 211
 212   async listVideosViewedForStats (hour: number) {
 213     const { setKey } = this.generateVideoViewStatsKeys({ hour })
 214
 215     const stringIds = await this.getSet(setKey)
 216     return stringIds.map(s => parseInt(s, 10))
 217   }
 218
 219   deleteVideoViewsStats (videoId: number, hour: number) {
 220     const { setKey, videoKey } = this.generateVideoViewStatsKeys({ videoId, hour })
 221
 222     return Promise.all([
 223       this.deleteFromSet(setKey, videoId.toString()),
 224       this.deleteKey(videoKey)
 225     ])
 226   }
 227
 228   /* ************ Local video views buffer ************ */
 229
 230   addLocalVideoView (videoId: number) {
 231     const { videoKey, setKey } = this.generateLocalVideoViewsKeys(videoId)
 232
 233     return Promise.all([
 234       this.addToSet(setKey, videoId.toString()),
 235       this.increment(videoKey)
 236     ])
 237   }
 238
 239   async getLocalVideoViews (videoId: number) {
 240     const { videoKey } = this.generateLocalVideoViewsKeys(videoId)
 241
 242     const valueString = await this.getValue(videoKey)
 243     const valueInt = parseInt(valueString, 10)
 244
 245     if (isNaN(valueInt)) {
 246       logger.error('Cannot get videos views of video %d: views number is NaN (%s).', videoId, valueString)
 247       return undefined
 248     }
 249
 250     return valueInt
 251   }
 252
 253   async listLocalVideosViewed () {
 254     const { setKey } = this.generateLocalVideoViewsKeys()
 255
 256     const stringIds = await this.getSet(setKey)
 257     return stringIds.map(s => parseInt(s, 10))
 258   }
 259
 260   deleteLocalVideoViews (videoId: number) {
 261     const { setKey, videoKey } = this.generateLocalVideoViewsKeys(videoId)
 262
 263     return Promise.all([
 264       this.deleteFromSet(setKey, videoId.toString()),
 265       this.deleteKey(videoKey)
 266     ])
 267   }
 268
 269   /* ************ Video viewers stats ************ */
 270
 271   getLocalVideoViewer (options: {
 272     key?: string
 273     // Or
 274     ip?: string
 275     videoId?: number
 276   }) {
 277     if (options.key) return this.getObject(options.key)
 278
 279     const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId)
 280
 281     return this.getObject(viewerKey)
 282   }
 283
 284   setLocalVideoViewer (ip: string, videoId: number, object: any) {
 285     const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId)
 286
 287     return Promise.all([
 288       this.addToSet(setKey, viewerKey),
 289       this.setObject(viewerKey, object)
 290     ])
 291   }
 292
 293   listLocalVideoViewerKeys () {
 294     const { setKey } = this.generateLocalVideoViewerKeys()
 295
 296     return this.getSet(setKey)
 297   }
 298
 299   deleteLocalVideoViewersKeys (key: string) {
 300     const { setKey } = this.generateLocalVideoViewerKeys()
 301
 302     return Promise.all([
 303       this.deleteFromSet(setKey, key),
 304       this.deleteKey(key)
 305     ])
 306   }
 307
 308   /* ************ Resumable uploads final responses ************ */
 309
 310   setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) {
 311     return this.setValue(
 312       'resumable-upload-' + uploadId,
 313       response
 314         ? JSON.stringify(response)
 315         : '',
 316       RESUMABLE_UPLOAD_SESSION_LIFETIME
 317     )
 318   }
 319
 320   doesUploadSessionExist (uploadId: string) {
 321     return this.exists('resumable-upload-' + uploadId)
 322   }
 323
 324   async getUploadSession (uploadId: string) {
 325     const value = await this.getValue('resumable-upload-' + uploadId)
 326
 327     return value
 328       ? JSON.parse(value)
 329       : ''
 330   }
 331
 332   deleteUploadSession (uploadId: string) {
 333     return this.deleteKey('resumable-upload-' + uploadId)
 334   }
 335
 336   /* ************ AP resource unavailability ************ */
 337
 338   async addAPUnavailability (url: string) {
 339     const key = this.generateAPUnavailabilityKey(url)
 340
 341     const value = await this.increment(key)
 342     await this.setExpiration(key, AP_CLEANER.PERIOD * 2)
 343
 344     return value
 345   }
 346
 347   /* ************ Keys generation ************ */
 348
 349   private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string }
 350   private generateLocalVideoViewsKeys (): { setKey: string }
 351   private generateLocalVideoViewsKeys (videoId?: number) {
 352     return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` }
 353   }
 354
 355   private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string }
 356   private generateLocalVideoViewerKeys (): { setKey: string }
 357   private generateLocalVideoViewerKeys (ip?: string, videoId?: number) {
 358     return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` }
 359   }
 360
 361   private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) {
 362     const hour = exists(options.hour)
 363       ? options.hour
 364       : new Date().getHours()
 365
 366     return { setKey: `videos-view-h${hour}`, videoKey: `video-view-${options.videoId}-h${hour}` }
 367   }
 368
 369   private generateResetPasswordKey (userId: number) {
 370     return 'reset-password-' + userId
 371   }
 372
 373   private generateTwoFactorRequestKey (userId: number, token: string) {
 374     return 'two-factor-request-' + userId + '-' + token
 375   }
 376
 377   private generateUserVerifyEmailKey (userId: number) {
 378     return 'verify-email-user-' + userId
 379   }
 380
 381   private generateRegistrationVerifyEmailKey (registrationId: number) {
 382     return 'verify-email-registration-' + registrationId
 383   }
 384
 385   private generateIPViewKey (ip: string, videoUUID: string) {
 386     return `views-${videoUUID}-${ip}`
 387   }
 388
 389   private generateContactFormKey (ip: string) {
 390     return 'contact-form-' + ip
 391   }
 392
 393   private generateAPUnavailabilityKey (url: string) {
 394     return 'ap-unavailability-' + sha256(url)
 395   }
 396
 397   /* ************ Redis helpers ************ */
 398
 399   private getValue (key: string) {
 400     return this.client.get(this.prefix + key)
 401   }
 402
 403   private getSet (key: string) {
 404     return this.client.smembers(this.prefix + key)
 405   }
 406
 407   private addToSet (key: string, value: string) {
 408     return this.client.sadd(this.prefix + key, value)
 409   }
 410
 411   private deleteFromSet (key: string, value: string) {
 412     return this.client.srem(this.prefix + key, value)
 413   }
 414
 415   private deleteKey (key: string) {
 416     return this.client.del(this.prefix + key)
 417   }
 418
 419   private async getObject (key: string) {
 420     const value = await this.getValue(key)
 421     if (!value) return null
 422
 423     return JSON.parse(value)
 424   }
 425
 426   private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) {
 427     return this.setValue(key, JSON.stringify(value), expirationMilliseconds)
 428   }
 429
 430   private async setValue (key: string, value: string, expirationMilliseconds?: number) {
 431     const result = expirationMilliseconds !== undefined
 432       ? await this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds)
 433       : await this.client.set(this.prefix + key, value)
 434
 435     if (result !== 'OK') throw new Error('Redis set result is not OK.')
 436   }
 437
 438   private removeValue (key: string) {
 439     return this.client.del(this.prefix + key)
 440   }
 441
 442   private increment (key: string) {
 443     return this.client.incr(this.prefix + key)
 444   }
 445
 446   private async exists (key: string) {
 447     const result = await this.client.exists(this.prefix + key)
 448
 449     return result !== 0
 450   }
 451
 452   private setExpiration (key: string, ms: number) {
 453     return this.client.expire(this.prefix + key, ms / 1000)
 454   }
 455
 456   static get Instance () {
 457     return this.instance || (this.instance = new this())
 458   }
 459 }
 460
 461 // ---------------------------------------------------------------------------
 462
 463 export {
 464   Redis
 465 }