server/lib/redis.ts

   1 import { createClient, RedisClientOptions, RedisModules, RedisScripts } from 'redis'
   2 import { exists } from '@server/helpers/custom-validators/misc'
   3 import { sha256 } from '@shared/extra-utils'
   4 import { logger } from '../helpers/logger'
   5 import { generateRandomString } from '../helpers/utils'
   6 import { CONFIG } from '../initializers/config'
   7 import {
   8   AP_CLEANER,
   9   CONTACT_FORM_LIFETIME,
  10   RESUMABLE_UPLOAD_SESSION_LIFETIME,
  11   TRACKER_RATE_LIMITS,
  12   USER_EMAIL_VERIFY_LIFETIME,
  13   USER_PASSWORD_CREATE_LIFETIME,
  14   USER_PASSWORD_RESET_LIFETIME,
  15   VIEW_LIFETIME,
  16   WEBSERVER
  17 } from '../initializers/constants'
  18
  19 // Only used for typings
  20 // TODO: remove when https://github.com/microsoft/TypeScript/issues/37181 is fixed
  21 const redisClientWrapperForType = () => createClient<{}, RedisScripts>()
  22
  23 class Redis {
  24
  25   private static instance: Redis
  26   private initialized = false
  27   private connected = false
  28   private client: ReturnType<typeof redisClientWrapperForType>
  29   private prefix: string
  30
  31   private constructor () {
  32   }
  33
  34   init () {
  35     // Already initialized
  36     if (this.initialized === true) return
  37     this.initialized = true
  38
  39     this.client = createClient(Redis.getRedisClientOptions())
  40
  41     logger.info('Connecting to redis...')
  42
  43     this.client.connect()
  44       .then(() => {
  45         logger.info('Connected to redis.')
  46
  47         this.connected = true
  48       }).catch(err => {
  49         logger.error('Cannot connect to redis', { err })
  50         process.exit(-1)
  51       })
  52
  53     this.prefix = 'redis-' + WEBSERVER.HOST + '-'
  54   }
  55
  56   static getRedisClientOptions () {
  57     let config: RedisClientOptions<RedisModules, {}> = {
  58       socket: {
  59         connectTimeout: 20000 // Could be slow since node use sync call to compile PeerTube
  60       }
  61     }
  62
  63     if (CONFIG.REDIS.AUTH) {
  64       config = { ...config, password: CONFIG.REDIS.AUTH }
  65     }
  66
  67     if (CONFIG.REDIS.DB) {
  68       config = { ...config, database: CONFIG.REDIS.DB }
  69     }
  70
  71     if (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT) {
  72       config.socket = { ...config.socket, host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT }
  73     } else {
  74       config.socket = { ...config.socket, path: CONFIG.REDIS.SOCKET }
  75     }
  76
  77     return config
  78   }
  79
  80   getClient () {
  81     return this.client
  82   }
  83
  84   getPrefix () {
  85     return this.prefix
  86   }
  87
  88   isConnected () {
  89     return this.connected
  90   }
  91
  92   /* ************ Forgot password ************ */
  93
  94   async setResetPasswordVerificationString (userId: number) {
  95     const generatedString = await generateRandomString(32)
  96
  97     await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_RESET_LIFETIME)
  98
  99     return generatedString
 100   }
 101
 102   async setCreatePasswordVerificationString (userId: number) {
 103     const generatedString = await generateRandomString(32)
 104
 105     await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_CREATE_LIFETIME)
 106
 107     return generatedString
 108   }
 109
 110   async removePasswordVerificationString (userId: number) {
 111     return this.removeValue(this.generateResetPasswordKey(userId))
 112   }
 113
 114   async getResetPasswordLink (userId: number) {
 115     return this.getValue(this.generateResetPasswordKey(userId))
 116   }
 117
 118   /* ************ Email verification ************ */
 119
 120   async setVerifyEmailVerificationString (userId: number) {
 121     const generatedString = await generateRandomString(32)
 122
 123     await this.setValue(this.generateVerifyEmailKey(userId), generatedString, USER_EMAIL_VERIFY_LIFETIME)
 124
 125     return generatedString
 126   }
 127
 128   async getVerifyEmailLink (userId: number) {
 129     return this.getValue(this.generateVerifyEmailKey(userId))
 130   }
 131
 132   /* ************ Contact form per IP ************ */
 133
 134   async setContactFormIp (ip: string) {
 135     return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME)
 136   }
 137
 138   async doesContactFormIpExist (ip: string) {
 139     return this.exists(this.generateContactFormKey(ip))
 140   }
 141
 142   /* ************ Views per IP ************ */
 143
 144   setIPVideoView (ip: string, videoUUID: string) {
 145     return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW)
 146   }
 147
 148   async doesVideoIPViewExist (ip: string, videoUUID: string) {
 149     return this.exists(this.generateIPViewKey(ip, videoUUID))
 150   }
 151
 152   /* ************ Tracker IP block ************ */
 153
 154   setTrackerBlockIP (ip: string) {
 155     return this.setValue(this.generateTrackerBlockIPKey(ip), '1', TRACKER_RATE_LIMITS.BLOCK_IP_LIFETIME)
 156   }
 157
 158   async doesTrackerBlockIPExist (ip: string) {
 159     return this.exists(this.generateTrackerBlockIPKey(ip))
 160   }
 161
 162   /* ************ Video views stats ************ */
 163
 164   addVideoViewStats (videoId: number) {
 165     const { videoKey, setKey } = this.generateVideoViewStatsKeys({ videoId })
 166
 167     return Promise.all([
 168       this.addToSet(setKey, videoId.toString()),
 169       this.increment(videoKey)
 170     ])
 171   }
 172
 173   async getVideoViewsStats (videoId: number, hour: number) {
 174     const { videoKey } = this.generateVideoViewStatsKeys({ videoId, hour })
 175
 176     const valueString = await this.getValue(videoKey)
 177     const valueInt = parseInt(valueString, 10)
 178
 179     if (isNaN(valueInt)) {
 180       logger.error('Cannot get videos views stats of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString)
 181       return undefined
 182     }
 183
 184     return valueInt
 185   }
 186
 187   async listVideosViewedForStats (hour: number) {
 188     const { setKey } = this.generateVideoViewStatsKeys({ hour })
 189
 190     const stringIds = await this.getSet(setKey)
 191     return stringIds.map(s => parseInt(s, 10))
 192   }
 193
 194   deleteVideoViewsStats (videoId: number, hour: number) {
 195     const { setKey, videoKey } = this.generateVideoViewStatsKeys({ videoId, hour })
 196
 197     return Promise.all([
 198       this.deleteFromSet(setKey, videoId.toString()),
 199       this.deleteKey(videoKey)
 200     ])
 201   }
 202
 203   /* ************ Local video views buffer ************ */
 204
 205   addLocalVideoView (videoId: number) {
 206     const { videoKey, setKey } = this.generateLocalVideoViewsKeys(videoId)
 207
 208     return Promise.all([
 209       this.addToSet(setKey, videoId.toString()),
 210       this.increment(videoKey)
 211     ])
 212   }
 213
 214   async getLocalVideoViews (videoId: number) {
 215     const { videoKey } = this.generateLocalVideoViewsKeys(videoId)
 216
 217     const valueString = await this.getValue(videoKey)
 218     const valueInt = parseInt(valueString, 10)
 219
 220     if (isNaN(valueInt)) {
 221       logger.error('Cannot get videos views of video %d: views number is NaN (%s).', videoId, valueString)
 222       return undefined
 223     }
 224
 225     return valueInt
 226   }
 227
 228   async listLocalVideosViewed () {
 229     const { setKey } = this.generateLocalVideoViewsKeys()
 230
 231     const stringIds = await this.getSet(setKey)
 232     return stringIds.map(s => parseInt(s, 10))
 233   }
 234
 235   deleteLocalVideoViews (videoId: number) {
 236     const { setKey, videoKey } = this.generateLocalVideoViewsKeys(videoId)
 237
 238     return Promise.all([
 239       this.deleteFromSet(setKey, videoId.toString()),
 240       this.deleteKey(videoKey)
 241     ])
 242   }
 243
 244   /* ************ Video viewers stats ************ */
 245
 246   getLocalVideoViewer (options: {
 247     key?: string
 248     // Or
 249     ip?: string
 250     videoId?: number
 251   }) {
 252     if (options.key) return this.getObject(options.key)
 253
 254     const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId)
 255
 256     return this.getObject(viewerKey)
 257   }
 258
 259   setLocalVideoViewer (ip: string, videoId: number, object: any) {
 260     const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId)
 261
 262     return Promise.all([
 263       this.addToSet(setKey, viewerKey),
 264       this.setObject(viewerKey, object)
 265     ])
 266   }
 267
 268   listLocalVideoViewerKeys () {
 269     const { setKey } = this.generateLocalVideoViewerKeys()
 270
 271     return this.getSet(setKey)
 272   }
 273
 274   deleteLocalVideoViewersKeys (key: string) {
 275     const { setKey } = this.generateLocalVideoViewerKeys()
 276
 277     return Promise.all([
 278       this.deleteFromSet(setKey, key),
 279       this.deleteKey(key)
 280     ])
 281   }
 282
 283   /* ************ Resumable uploads final responses ************ */
 284
 285   setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) {
 286     return this.setValue(
 287       'resumable-upload-' + uploadId,
 288       response
 289         ? JSON.stringify(response)
 290         : '',
 291       RESUMABLE_UPLOAD_SESSION_LIFETIME
 292     )
 293   }
 294
 295   doesUploadSessionExist (uploadId: string) {
 296     return this.exists('resumable-upload-' + uploadId)
 297   }
 298
 299   async getUploadSession (uploadId: string) {
 300     const value = await this.getValue('resumable-upload-' + uploadId)
 301
 302     return value
 303       ? JSON.parse(value)
 304       : ''
 305   }
 306
 307   deleteUploadSession (uploadId: string) {
 308     return this.deleteKey('resumable-upload-' + uploadId)
 309   }
 310
 311   /* ************ AP resource unavailability ************ */
 312
 313   async addAPUnavailability (url: string) {
 314     const key = this.generateAPUnavailabilityKey(url)
 315
 316     const value = await this.increment(key)
 317     await this.setExpiration(key, AP_CLEANER.PERIOD * 2)
 318
 319     return value
 320   }
 321
 322   /* ************ Keys generation ************ */
 323
 324   private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string }
 325   private generateLocalVideoViewsKeys (): { setKey: string }
 326   private generateLocalVideoViewsKeys (videoId?: number) {
 327     return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` }
 328   }
 329
 330   private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string }
 331   private generateLocalVideoViewerKeys (): { setKey: string }
 332   private generateLocalVideoViewerKeys (ip?: string, videoId?: number) {
 333     return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` }
 334   }
 335
 336   private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) {
 337     const hour = exists(options.hour)
 338       ? options.hour
 339       : new Date().getHours()
 340
 341     return { setKey: `videos-view-h${hour}`, videoKey: `video-view-${options.videoId}-h${hour}` }
 342   }
 343
 344   private generateResetPasswordKey (userId: number) {
 345     return 'reset-password-' + userId
 346   }
 347
 348   private generateVerifyEmailKey (userId: number) {
 349     return 'verify-email-' + userId
 350   }
 351
 352   private generateIPViewKey (ip: string, videoUUID: string) {
 353     return `views-${videoUUID}-${ip}`
 354   }
 355
 356   private generateTrackerBlockIPKey (ip: string) {
 357     return `tracker-block-ip-${ip}`
 358   }
 359
 360   private generateContactFormKey (ip: string) {
 361     return 'contact-form-' + ip
 362   }
 363
 364   private generateAPUnavailabilityKey (url: string) {
 365     return 'ap-unavailability-' + sha256(url)
 366   }
 367
 368   /* ************ Redis helpers ************ */
 369
 370   private getValue (key: string) {
 371     return this.client.get(this.prefix + key)
 372   }
 373
 374   private getSet (key: string) {
 375     return this.client.sMembers(this.prefix + key)
 376   }
 377
 378   private addToSet (key: string, value: string) {
 379     return this.client.sAdd(this.prefix + key, value)
 380   }
 381
 382   private deleteFromSet (key: string, value: string) {
 383     return this.client.sRem(this.prefix + key, value)
 384   }
 385
 386   private deleteKey (key: string) {
 387     return this.client.del(this.prefix + key)
 388   }
 389
 390   private async getObject (key: string) {
 391     const value = await this.getValue(key)
 392     if (!value) return null
 393
 394     return JSON.parse(value)
 395   }
 396
 397   private setObject (key: string, value: { [ id: string ]: number | string }) {
 398     return this.setValue(key, JSON.stringify(value))
 399   }
 400
 401   private async setValue (key: string, value: string, expirationMilliseconds?: number) {
 402     const options = expirationMilliseconds
 403       ? { PX: expirationMilliseconds }
 404       : {}
 405
 406     const result = await this.client.set(this.prefix + key, value, options)
 407
 408     if (result !== 'OK') throw new Error('Redis set result is not OK.')
 409   }
 410
 411   private removeValue (key: string) {
 412     return this.client.del(this.prefix + key)
 413   }
 414
 415   private increment (key: string) {
 416     return this.client.incr(this.prefix + key)
 417   }
 418
 419   private async exists (key: string) {
 420     const result = await this.client.exists(this.prefix + key)
 421
 422     return result !== 0
 423   }
 424
 425   private setExpiration (key: string, ms: number) {
 426     return this.client.expire(this.prefix + key, ms / 1000)
 427   }
 428
 429   static get Instance () {
 430     return this.instance || (this.instance = new this())
 431   }
 432 }
 433
 434 // ---------------------------------------------------------------------------
 435
 436 export {
 437   Redis
 438 }