aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-03-19 14:23:17 +0100
committerChocobozzz <me@florianbigard.com>2019-03-19 14:30:43 +0100
commitd74d29ad9e35929491cf37223398d2535ab23de0 (patch)
tree2812c9acbc05be0603eb671f8e6bd81086cf84d5
parent9f79ade627f0044606a9fbbe16ca0154661d12b9 (diff)
downloadPeerTube-d74d29ad9e35929491cf37223398d2535ab23de0.tar.gz
PeerTube-d74d29ad9e35929491cf37223398d2535ab23de0.tar.zst
PeerTube-d74d29ad9e35929491cf37223398d2535ab23de0.zip
Limit user tokens cache
-rw-r--r--server.ts8
-rw-r--r--server/controllers/static.ts3
-rw-r--r--server/initializers/constants.ts13
-rw-r--r--server/initializers/installer.ts10
-rw-r--r--server/lib/files-cache/abstract-video-static-file-cache.ts (renamed from server/lib/cache/abstract-video-static-file-cache.ts)0
-rw-r--r--server/lib/files-cache/actor-follow-score-cache.ts (renamed from server/lib/cache/actor-follow-score-cache.ts)0
-rw-r--r--server/lib/files-cache/index.ts (renamed from server/lib/cache/index.ts)0
-rw-r--r--server/lib/files-cache/videos-caption-cache.ts (renamed from server/lib/cache/videos-caption-cache.ts)4
-rw-r--r--server/lib/files-cache/videos-preview-cache.ts (renamed from server/lib/cache/videos-preview-cache.ts)4
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-broadcast.ts3
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-unicast.ts2
-rw-r--r--server/lib/oauth-model.ts12
-rw-r--r--server/lib/schedulers/actor-follow-scheduler.ts2
13 files changed, 36 insertions, 25 deletions
diff --git a/server.ts b/server.ts
index 9fe741175..df56bcd82 100644
--- a/server.ts
+++ b/server.ts
@@ -28,7 +28,7 @@ import { checkMissedConfig, checkFFmpeg } from './server/initializers/checker-be
28 28
29// Do not use barrels because we don't want to load all modules here (we need to initialize database first) 29// Do not use barrels because we don't want to load all modules here (we need to initialize database first)
30import { logger } from './server/helpers/logger' 30import { logger } from './server/helpers/logger'
31import { API_VERSION, CONFIG, CACHE } from './server/initializers/constants' 31import { API_VERSION, CONFIG, FILES_CACHE } from './server/initializers/constants'
32 32
33const missed = checkMissedConfig() 33const missed = checkMissedConfig()
34if (missed.length !== 0) { 34if (missed.length !== 0) {
@@ -82,7 +82,7 @@ migrate()
82import { installApplication } from './server/initializers' 82import { installApplication } from './server/initializers'
83import { Emailer } from './server/lib/emailer' 83import { Emailer } from './server/lib/emailer'
84import { JobQueue } from './server/lib/job-queue' 84import { JobQueue } from './server/lib/job-queue'
85import { VideosPreviewCache, VideosCaptionCache } from './server/lib/cache' 85import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
86import { 86import {
87 activityPubRouter, 87 activityPubRouter,
88 apiRouter, 88 apiRouter,
@@ -218,8 +218,8 @@ async function startApplication () {
218 ]) 218 ])
219 219
220 // Caches initializations 220 // Caches initializations
221 VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, CACHE.PREVIEWS.MAX_AGE) 221 VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
222 VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, CACHE.VIDEO_CAPTIONS.MAX_AGE) 222 VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
223 223
224 // Enable Schedulers 224 // Enable Schedulers
225 ActorFollowScheduler.Instance.enable() 225 ActorFollowScheduler.Instance.enable()
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 7b14320e4..e65c7afd3 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -8,11 +8,10 @@ import {
8 STATIC_MAX_AGE, 8 STATIC_MAX_AGE,
9 STATIC_PATHS 9 STATIC_PATHS
10} from '../initializers' 10} from '../initializers'
11import { VideosPreviewCache } from '../lib/cache' 11import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
12import { cacheRoute } from '../middlewares/cache' 12import { cacheRoute } from '../middlewares/cache'
13import { asyncMiddleware, videosGetValidator } from '../middlewares' 13import { asyncMiddleware, videosGetValidator } from '../middlewares'
14import { VideoModel } from '../models/video/video' 14import { VideoModel } from '../models/video/video'
15import { VideosCaptionCache } from '../lib/cache/videos-caption-cache'
16import { UserModel } from '../models/account/user' 15import { UserModel } from '../models/account/user'
17import { VideoCommentModel } from '../models/video/video-comment' 16import { VideoCommentModel } from '../models/video/video-comment'
18import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo' 17import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo'
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7fac8a4d6..7a3ec3874 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -660,7 +660,7 @@ const EMBED_SIZE = {
660} 660}
661 661
662// Sub folders of cache directory 662// Sub folders of cache directory
663const CACHE = { 663const FILES_CACHE = {
664 PREVIEWS: { 664 PREVIEWS: {
665 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'), 665 DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
666 MAX_AGE: 1000 * 3600 * 3 // 3 hours 666 MAX_AGE: 1000 * 3600 * 3 // 3 hours
@@ -671,6 +671,12 @@ const CACHE = {
671 } 671 }
672} 672}
673 673
674const CACHE = {
675 USER_TOKENS: {
676 MAX_SIZE: 10000
677 }
678}
679
674const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls') 680const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
675const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') 681const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
676 682
@@ -741,7 +747,7 @@ if (isTestInstance() === true) {
741 747
742 JOB_ATTEMPTS['email'] = 1 748 JOB_ATTEMPTS['email'] = 1
743 749
744 CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 750 FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
745 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 751 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
746 ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' 752 ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
747 753
@@ -759,7 +765,7 @@ export {
759 ACCEPT_HEADERS, 765 ACCEPT_HEADERS,
760 BCRYPT_SALT_SIZE, 766 BCRYPT_SALT_SIZE,
761 TRACKER_RATE_LIMITS, 767 TRACKER_RATE_LIMITS,
762 CACHE, 768 FILES_CACHE,
763 CONFIG, 769 CONFIG,
764 CONSTRAINTS_FIELDS, 770 CONSTRAINTS_FIELDS,
765 EMBED_SIZE, 771 EMBED_SIZE,
@@ -799,6 +805,7 @@ export {
799 VIDEO_TRANSCODING_FPS, 805 VIDEO_TRANSCODING_FPS,
800 FFMPEG_NICE, 806 FFMPEG_NICE,
801 VIDEO_ABUSE_STATES, 807 VIDEO_ABUSE_STATES,
808 CACHE,
802 JOB_REQUEST_TIMEOUT, 809 JOB_REQUEST_TIMEOUT,
803 USER_PASSWORD_RESET_LIFETIME, 810 USER_PASSWORD_RESET_LIFETIME,
804 MEMOIZE_TTL, 811 MEMOIZE_TTL,
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index cd2c942fd..07af96b68 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
6import { ApplicationModel } from '../models/application/application' 6import { ApplicationModel } from '../models/application/application'
7import { OAuthClientModel } from '../models/oauth/oauth-client' 7import { OAuthClientModel } from '../models/oauth/oauth-client'
8import { applicationExist, clientsExist, usersExist } from './checker-after-init' 8import { applicationExist, clientsExist, usersExist } from './checker-after-init'
9import { CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' 9import { FILES_CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
10import { sequelizeTypescript } from './database' 10import { sequelizeTypescript } from './database'
11import { remove, ensureDir } from 'fs-extra' 11import { remove, ensureDir } from 'fs-extra'
12 12
@@ -42,8 +42,8 @@ export {
42// --------------------------------------------------------------------------- 42// ---------------------------------------------------------------------------
43 43
44function removeCacheAndTmpDirectories () { 44function removeCacheAndTmpDirectories () {
45 const cacheDirectories = Object.keys(CACHE) 45 const cacheDirectories = Object.keys(FILES_CACHE)
46 .map(k => CACHE[k].DIRECTORY) 46 .map(k => FILES_CACHE[k].DIRECTORY)
47 47
48 const tasks: Promise<any>[] = [] 48 const tasks: Promise<any>[] = []
49 49
@@ -60,8 +60,8 @@ function removeCacheAndTmpDirectories () {
60 60
61function createDirectoriesIfNotExist () { 61function createDirectoriesIfNotExist () {
62 const storage = CONFIG.STORAGE 62 const storage = CONFIG.STORAGE
63 const cacheDirectories = Object.keys(CACHE) 63 const cacheDirectories = Object.keys(FILES_CACHE)
64 .map(k => CACHE[k].DIRECTORY) 64 .map(k => FILES_CACHE[k].DIRECTORY)
65 65
66 const tasks: Promise<void>[] = [] 66 const tasks: Promise<void>[] = []
67 for (const key of Object.keys(storage)) { 67 for (const key of Object.keys(storage)) {
diff --git a/server/lib/cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts
index 7512f2b9d..7512f2b9d 100644
--- a/server/lib/cache/abstract-video-static-file-cache.ts
+++ b/server/lib/files-cache/abstract-video-static-file-cache.ts
diff --git a/server/lib/cache/actor-follow-score-cache.ts b/server/lib/files-cache/actor-follow-score-cache.ts
index d070bde09..d070bde09 100644
--- a/server/lib/cache/actor-follow-score-cache.ts
+++ b/server/lib/files-cache/actor-follow-score-cache.ts
diff --git a/server/lib/cache/index.ts b/server/lib/files-cache/index.ts
index e921d04a7..e921d04a7 100644
--- a/server/lib/cache/index.ts
+++ b/server/lib/files-cache/index.ts
diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts
index f240affbc..fe5b441af 100644
--- a/server/lib/cache/videos-caption-cache.ts
+++ b/server/lib/files-cache/videos-caption-cache.ts
@@ -1,5 +1,5 @@
1import { join } from 'path' 1import { join } from 'path'
2import { CACHE, CONFIG } from '../../initializers' 2import { FILES_CACHE, CONFIG } from '../../initializers'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { VideoCaptionModel } from '../../models/video/video-caption' 4import { VideoCaptionModel } from '../../models/video/video-caption'
5import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 5import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
@@ -42,7 +42,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
42 if (!video) return undefined 42 if (!video) return undefined
43 43
44 const remoteStaticPath = videoCaption.getCaptionStaticPath() 44 const remoteStaticPath = videoCaption.getCaptionStaticPath()
45 const destPath = join(CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) 45 const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
46 46
47 return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) 47 return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
48 } 48 }
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts
index a5d6f5b62..01cd3647e 100644
--- a/server/lib/cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/videos-preview-cache.ts
@@ -1,5 +1,5 @@
1import { join } from 'path' 1import { join } from 'path'
2import { CACHE, CONFIG, STATIC_PATHS } from '../../initializers' 2import { FILES_CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
5 5
@@ -31,7 +31,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
31 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') 31 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
32 32
33 const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) 33 const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
34 const destPath = join(CACHE.PREVIEWS.DIRECTORY, video.getPreviewName()) 34 const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
35 35
36 return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) 36 return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
37 } 37 }
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index 9493945ff..2b1e21c39 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -2,10 +2,9 @@ import * as Bull from 'bull'
2import * as Bluebird from 'bluebird' 2import * as Bluebird from 'bluebird'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { doRequest } from '../../../helpers/requests' 4import { doRequest } from '../../../helpers/requests'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' 5import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
7import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' 6import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers'
8import { ActorFollowScoreCache } from '../../cache' 7import { ActorFollowScoreCache } from '../../files-cache'
9 8
10export type ActivitypubHttpBroadcastPayload = { 9export type ActivitypubHttpBroadcastPayload = {
11 uris: string[] 10 uris: string[]
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
index 3973dcdc8..59de7119a 100644
--- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
@@ -3,7 +3,7 @@ import { logger } from '../../../helpers/logger'
3import { doRequest } from '../../../helpers/requests' 3import { doRequest } from '../../../helpers/requests'
4import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' 4import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
5import { JOB_REQUEST_TIMEOUT } from '../../../initializers' 5import { JOB_REQUEST_TIMEOUT } from '../../../initializers'
6import { ActorFollowScoreCache } from '../../cache' 6import { ActorFollowScoreCache } from '../../files-cache'
7 7
8export type ActivitypubHttpUnicastPayload = { 8export type ActivitypubHttpUnicastPayload = {
9 uri: string 9 uri: string
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 2cd2ae97c..5b4a2bcf9 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -4,12 +4,12 @@ import { logger } from '../helpers/logger'
4import { UserModel } from '../models/account/user' 4import { UserModel } from '../models/account/user'
5import { OAuthClientModel } from '../models/oauth/oauth-client' 5import { OAuthClientModel } from '../models/oauth/oauth-client'
6import { OAuthTokenModel } from '../models/oauth/oauth-token' 6import { OAuthTokenModel } from '../models/oauth/oauth-token'
7import { CONFIG } from '../initializers/constants' 7import { CONFIG, CACHE } from '../initializers/constants'
8import { Transaction } from 'sequelize' 8import { Transaction } from 'sequelize'
9 9
10type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } 10type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
11const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} 11let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
12const userHavingToken: { [ userId: number ]: string } = {} 12let userHavingToken: { [ userId: number ]: string } = {}
13 13
14// --------------------------------------------------------------------------- 14// ---------------------------------------------------------------------------
15 15
@@ -43,6 +43,12 @@ function getAccessToken (bearerToken: string) {
43 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) 43 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
44 .then(tokenModel => { 44 .then(tokenModel => {
45 if (tokenModel) { 45 if (tokenModel) {
46 // Reinit our cache
47 if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) {
48 accessTokenCache = {}
49 userHavingToken = {}
50 }
51
46 accessTokenCache[ bearerToken ] = tokenModel 52 accessTokenCache[ bearerToken ] = tokenModel
47 userHavingToken[ tokenModel.userId ] = tokenModel.accessToken 53 userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
48 } 54 }
diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts
index 3967be7f8..05e6bd139 100644
--- a/server/lib/schedulers/actor-follow-scheduler.ts
+++ b/server/lib/schedulers/actor-follow-scheduler.ts
@@ -3,7 +3,7 @@ import { logger } from '../../helpers/logger'
3import { ActorFollowModel } from '../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../models/activitypub/actor-follow'
4import { AbstractScheduler } from './abstract-scheduler' 4import { AbstractScheduler } from './abstract-scheduler'
5import { SCHEDULER_INTERVALS_MS } from '../../initializers' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers'
6import { ActorFollowScoreCache } from '../cache' 6import { ActorFollowScoreCache } from '../files-cache'
7 7
8export class ActorFollowScheduler extends AbstractScheduler { 8export class ActorFollowScheduler extends AbstractScheduler {
9 9