server/lib/redis.ts

   1 import { createClient, RedisClientOptions, RedisModules, RedisScripts } from 'redis'
   2 import { exists } from '@server/helpers/custom-validators/misc'
   3 import { sha256 } from '@shared/extra-utils'
   4 import { logger } from '../helpers/logger'
   5 import { generateRandomString } from '../helpers/utils'
   6 import { CONFIG } from '../initializers/config'
   7 import {
   8   AP_CLEANER,
   9   CONTACT_FORM_LIFETIME,
  10   RESUMABLE_UPLOAD_SESSION_LIFETIME,
  11   TRACKER_RATE_LIMITS,
  12   USER_EMAIL_VERIFY_LIFETIME,
  13   USER_PASSWORD_CREATE_LIFETIME,
  14   USER_PASSWORD_RESET_LIFETIME,
  15   VIEW_LIFETIME,
  16   WEBSERVER
  17 } from '../initializers/constants'
  18
  19 // Only used for typings
  20 // TODO: remove when https://github.com/microsoft/TypeScript/issues/37181 is fixed
  21 const redisClientWrapperForType = () => createClient<{}, RedisScripts>()
  22
  23 class Redis {
  24
  25   private static instance: Redis
  26   private initialized = false
  27   private connected = false
  28   private client: ReturnType<typeof redisClientWrapperForType>
  29   private prefix: string
  30
  31   private constructor () {
  32   }
  33
  34   init () {
  35     // Already initialized
  36     if (this.initialized === true) return
  37     this.initialized = true
  38
  39     this.client = createClient(Redis.getRedisClientOptions())
  40
  41     logger.info('Connecting to redis...')
  42
  43     this.client.connect()
  44       .then(() => {
  45         logger.info('Connected to redis.')
  46
  47         this.connected = true
  48       }).catch(err => {
  49         logger.error('Cannot connect to redis', { err })
  50         process.exit(-1)
  51       })
  52
  53     this.prefix = 'redis-' + WEBSERVER.HOST + '-'
  54   }
  55
  56   static getRedisClientOptions () {
  57     let config: RedisClientOptions<RedisModules, {}> = {
  58       socket: {
  59         connectTimeout: 20000 // Could be slow since node use sync call to compile PeerTube
  60       }
  61     }
  62
  63     if (CONFIG.REDIS.AUTH) {
  64       config = { ...config, password: CONFIG.REDIS.AUTH }
  65     }
  66
  67     if (CONFIG.REDIS.DB) {
  68       config = { ...config, database: CONFIG.REDIS.DB }
  69     }
  70
  71     if (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT) {
  72       config.socket = { ...config.socket, host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT }
  73     } else {
  74       config.socket = { ...config.socket, path: CONFIG.REDIS.SOCKET }
  75     }
  76
  77     return config
  78   }
  79
  80   getClient () {
  81     return this.client
  82   }
  83
  84   getPrefix () {
  85     return this.prefix
  86   }
  87
  88   isConnected () {
  89     return this.connected
  90   }
  91
  92   /* ************ Forgot password ************ */
  93
  94   async setResetPasswordVerificationString (userId: number) {
  95     const generatedString = await generateRandomString(32)
  96
  97     await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_RESET_LIFETIME)
  98
  99     return generatedString
 100   }
 101
 102   async setCreatePasswordVerificationString (userId: number) {
 103     const generatedString = await generateRandomString(32)
 104
 105     await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_CREATE_LIFETIME)
 106
 107     return generatedString
 108   }
 109
 110   async removePasswordVerificationString (userId: number) {
 111     return this.removeValue(this.generateResetPasswordKey(userId))
 112   }
 113
 114   async getResetPasswordLink (userId: number) {
 115     return this.getValue(this.generateResetPasswordKey(userId))
 116   }
 117
 118   /* ************ Email verification ************ */
 119
 120   async setVerifyEmailVerificationString (userId: number) {
 121     const generatedString = await generateRandomString(32)
 122
 123     await this.setValue(this.generateVerifyEmailKey(userId), generatedString, USER_EMAIL_VERIFY_LIFETIME)
 124
 125     return generatedString
 126   }
 127
 128   async getVerifyEmailLink (userId: number) {
 129     return this.getValue(this.generateVerifyEmailKey(userId))
 130   }
 131
 132   /* ************ Contact form per IP ************ */
 133
 134   async setContactFormIp (ip: string) {
 135     return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME)
 136   }
 137
 138   async doesContactFormIpExist (ip: string) {
 139     return this.exists(this.generateContactFormKey(ip))
 140   }
 141
 142   /* ************ Views per IP ************ */
 143
 144   setIPVideoView (ip: string, videoUUID: string) {
 145     return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW)
 146   }
 147
 148   setIPVideoViewer (ip: string, videoUUID: string) {
 149     return this.setValue(this.generateIPViewerKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEWER)
 150   }
 151
 152   async doesVideoIPViewExist (ip: string, videoUUID: string) {
 153     return this.exists(this.generateIPViewKey(ip, videoUUID))
 154   }
 155
 156   async doesVideoIPViewerExist (ip: string, videoUUID: string) {
 157     return this.exists(this.generateIPViewerKey(ip, videoUUID))
 158   }
 159
 160   /* ************ Tracker IP block ************ */
 161
 162   setTrackerBlockIP (ip: string) {
 163     return this.setValue(this.generateTrackerBlockIPKey(ip), '1', TRACKER_RATE_LIMITS.BLOCK_IP_LIFETIME)
 164   }
 165
 166   async doesTrackerBlockIPExist (ip: string) {
 167     return this.exists(this.generateTrackerBlockIPKey(ip))
 168   }
 169
 170   /* ************ Video views stats ************ */
 171
 172   addVideoViewStats (videoId: number) {
 173     const { videoKey, setKey } = this.generateVideoViewStatsKeys({ videoId })
 174
 175     return Promise.all([
 176       this.addToSet(setKey, videoId.toString()),
 177       this.increment(videoKey)
 178     ])
 179   }
 180
 181   async getVideoViewsStats (videoId: number, hour: number) {
 182     const { videoKey } = this.generateVideoViewStatsKeys({ videoId, hour })
 183
 184     const valueString = await this.getValue(videoKey)
 185     const valueInt = parseInt(valueString, 10)
 186
 187     if (isNaN(valueInt)) {
 188       logger.error('Cannot get videos views stats of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString)
 189       return undefined
 190     }
 191
 192     return valueInt
 193   }
 194
 195   async listVideosViewedForStats (hour: number) {
 196     const { setKey } = this.generateVideoViewStatsKeys({ hour })
 197
 198     const stringIds = await this.getSet(setKey)
 199     return stringIds.map(s => parseInt(s, 10))
 200   }
 201
 202   deleteVideoViewsStats (videoId: number, hour: number) {
 203     const { setKey, videoKey } = this.generateVideoViewStatsKeys({ videoId, hour })
 204
 205     return Promise.all([
 206       this.deleteFromSet(setKey, videoId.toString()),
 207       this.deleteKey(videoKey)
 208     ])
 209   }
 210
 211   /* ************ Local video views buffer ************ */
 212
 213   addLocalVideoView (videoId: number) {
 214     const { videoKey, setKey } = this.generateLocalVideoViewsKeys(videoId)
 215
 216     return Promise.all([
 217       this.addToSet(setKey, videoId.toString()),
 218       this.increment(videoKey)
 219     ])
 220   }
 221
 222   async getLocalVideoViews (videoId: number) {
 223     const { videoKey } = this.generateLocalVideoViewsKeys(videoId)
 224
 225     const valueString = await this.getValue(videoKey)
 226     const valueInt = parseInt(valueString, 10)
 227
 228     if (isNaN(valueInt)) {
 229       logger.error('Cannot get videos views of video %d: views number is NaN (%s).', videoId, valueString)
 230       return undefined
 231     }
 232
 233     return valueInt
 234   }
 235
 236   async listLocalVideosViewed () {
 237     const { setKey } = this.generateLocalVideoViewsKeys()
 238
 239     const stringIds = await this.getSet(setKey)
 240     return stringIds.map(s => parseInt(s, 10))
 241   }
 242
 243   deleteLocalVideoViews (videoId: number) {
 244     const { setKey, videoKey } = this.generateLocalVideoViewsKeys(videoId)
 245
 246     return Promise.all([
 247       this.deleteFromSet(setKey, videoId.toString()),
 248       this.deleteKey(videoKey)
 249     ])
 250   }
 251
 252   /* ************ Video viewers stats ************ */
 253
 254   getLocalVideoViewer (options: {
 255     key?: string
 256     // Or
 257     ip?: string
 258     videoId?: number
 259   }) {
 260     if (options.key) return this.getObject(options.key)
 261
 262     const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId)
 263
 264     return this.getObject(viewerKey)
 265   }
 266
 267   setLocalVideoViewer (ip: string, videoId: number, object: any) {
 268     const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId)
 269
 270     return Promise.all([
 271       this.addToSet(setKey, viewerKey),
 272       this.setObject(viewerKey, object)
 273     ])
 274   }
 275
 276   listLocalVideoViewerKeys () {
 277     const { setKey } = this.generateLocalVideoViewerKeys()
 278
 279     return this.getSet(setKey)
 280   }
 281
 282   deleteLocalVideoViewersKeys (key: string) {
 283     const { setKey } = this.generateLocalVideoViewerKeys()
 284
 285     return Promise.all([
 286       this.deleteFromSet(setKey, key),
 287       this.deleteKey(key)
 288     ])
 289   }
 290
 291   /* ************ Resumable uploads final responses ************ */
 292
 293   setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) {
 294     return this.setValue(
 295       'resumable-upload-' + uploadId,
 296       response
 297         ? JSON.stringify(response)
 298         : '',
 299       RESUMABLE_UPLOAD_SESSION_LIFETIME
 300     )
 301   }
 302
 303   doesUploadSessionExist (uploadId: string) {
 304     return this.exists('resumable-upload-' + uploadId)
 305   }
 306
 307   async getUploadSession (uploadId: string) {
 308     const value = await this.getValue('resumable-upload-' + uploadId)
 309
 310     return value
 311       ? JSON.parse(value)
 312       : ''
 313   }
 314
 315   deleteUploadSession (uploadId: string) {
 316     return this.deleteKey('resumable-upload-' + uploadId)
 317   }
 318
 319   /* ************ AP ressource unavailability ************ */
 320
 321   async addAPUnavailability (url: string) {
 322     const key = this.generateAPUnavailabilityKey(url)
 323
 324     const value = await this.increment(key)
 325     await this.setExpiration(key, AP_CLEANER.PERIOD * 2)
 326
 327     return value
 328   }
 329
 330   /* ************ Keys generation ************ */
 331
 332   private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string }
 333   private generateLocalVideoViewsKeys (): { setKey: string }
 334   private generateLocalVideoViewsKeys (videoId?: number) {
 335     return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` }
 336   }
 337
 338   private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string }
 339   private generateLocalVideoViewerKeys (): { setKey: string }
 340   private generateLocalVideoViewerKeys (ip?: string, videoId?: number) {
 341     return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` }
 342   }
 343
 344   private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) {
 345     const hour = exists(options.hour)
 346       ? options.hour
 347       : new Date().getHours()
 348
 349     return { setKey: `videos-view-h${hour}`, videoKey: `video-view-${options.videoId}-h${hour}` }
 350   }
 351
 352   private generateResetPasswordKey (userId: number) {
 353     return 'reset-password-' + userId
 354   }
 355
 356   private generateVerifyEmailKey (userId: number) {
 357     return 'verify-email-' + userId
 358   }
 359
 360   private generateIPViewKey (ip: string, videoUUID: string) {
 361     return `views-${videoUUID}-${ip}`
 362   }
 363
 364   private generateIPViewerKey (ip: string, videoUUID: string) {
 365     return `viewer-${videoUUID}-${ip}`
 366   }
 367
 368   private generateTrackerBlockIPKey (ip: string) {
 369     return `tracker-block-ip-${ip}`
 370   }
 371
 372   private generateContactFormKey (ip: string) {
 373     return 'contact-form-' + ip
 374   }
 375
 376   private generateAPUnavailabilityKey (url: string) {
 377     return 'ap-unavailability-' + sha256(url)
 378   }
 379
 380   /* ************ Redis helpers ************ */
 381
 382   private getValue (key: string) {
 383     return this.client.get(this.prefix + key)
 384   }
 385
 386   private getSet (key: string) {
 387     return this.client.sMembers(this.prefix + key)
 388   }
 389
 390   private addToSet (key: string, value: string) {
 391     return this.client.sAdd(this.prefix + key, value)
 392   }
 393
 394   private deleteFromSet (key: string, value: string) {
 395     return this.client.sRem(this.prefix + key, value)
 396   }
 397
 398   private deleteKey (key: string) {
 399     return this.client.del(this.prefix + key)
 400   }
 401
 402   private async getObject (key: string) {
 403     const value = await this.getValue(key)
 404     if (!value) return null
 405
 406     return JSON.parse(value)
 407   }
 408
 409   private setObject (key: string, value: { [ id: string ]: number | string }) {
 410     return this.setValue(key, JSON.stringify(value))
 411   }
 412
 413   private async setValue (key: string, value: string, expirationMilliseconds?: number) {
 414     const options = expirationMilliseconds
 415       ? { PX: expirationMilliseconds }
 416       : {}
 417
 418     const result = await this.client.set(this.prefix + key, value, options)
 419
 420     if (result !== 'OK') throw new Error('Redis set result is not OK.')
 421   }
 422
 423   private removeValue (key: string) {
 424     return this.client.del(this.prefix + key)
 425   }
 426
 427   private increment (key: string) {
 428     return this.client.incr(this.prefix + key)
 429   }
 430
 431   private async exists (key: string) {
 432     const result = await this.client.exists(this.prefix + key)
 433
 434     return result !== 0
 435   }
 436
 437   private setExpiration (key: string, ms: number) {
 438     return this.client.expire(this.prefix + key, ms / 1000)
 439   }
 440
 441   static get Instance () {
 442     return this.instance || (this.instance = new this())
 443   }
 444 }
 445
 446 // ---------------------------------------------------------------------------
 447
 448 export {
 449   Redis
 450 }