From d74d29ad9e35929491cf37223398d2535ab23de0 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Tue, 19 Mar 2019 14:23:17 +0100
Subject: Limit user tokens cache
---
.../abstract-video-static-file-cache.ts | 52 +++++++++++++++++++++
server/lib/files-cache/actor-follow-score-cache.ts | 46 +++++++++++++++++++
server/lib/files-cache/index.ts | 3 ++
server/lib/files-cache/videos-caption-cache.ts | 53 ++++++++++++++++++++++
server/lib/files-cache/videos-preview-cache.ts | 42 +++++++++++++++++
5 files changed, 196 insertions(+)
create mode 100644 server/lib/files-cache/abstract-video-static-file-cache.ts
create mode 100644 server/lib/files-cache/actor-follow-score-cache.ts
create mode 100644 server/lib/files-cache/index.ts
create mode 100644 server/lib/files-cache/videos-caption-cache.ts
create mode 100644 server/lib/files-cache/videos-preview-cache.ts
(limited to 'server/lib/files-cache')
diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts
new file mode 100644
index 000000000..7512f2b9d
--- /dev/null
+++ b/server/lib/files-cache/abstract-video-static-file-cache.ts
@@ -0,0 +1,52 @@
+import * as AsyncLRU from 'async-lru'
+import { createWriteStream, remove } from 'fs-extra'
+import { logger } from '../../helpers/logger'
+import { VideoModel } from '../../models/video/video'
+import { fetchRemoteVideoStaticFile } from '../activitypub'
+
+export abstract class AbstractVideoStaticFileCache <T> {
+
+ protected lru
+
+ abstract getFilePath (params: T): Promise<string>
+
+ // Load and save the remote file, then return the local path from filesystem
+ protected abstract loadRemoteFile (key: string): Promise<string>
+
+ init (max: number, maxAge: number) {
+ this.lru = new AsyncLRU({
+ max,
+ maxAge,
+ load: (key, cb) => {
+ this.loadRemoteFile(key)
+ .then(res => cb(null, res))
+ .catch(err => cb(err))
+ }
+ })
+
+ this.lru.on('evict', (obj: { key: string, value: string }) => {
+ remove(obj.value)
+ .then(() => logger.debug('%s evicted from %s', obj.value, this.constructor.name))
+ })
+ }
+
+ protected loadFromLRU (key: string) {
+ return new Promise<string>((res, rej) => {
+ this.lru.get(key, (err, value) => {
+ err ? rej(err) : res(value)
+ })
+ })
+ }
+
+ protected saveRemoteVideoFileAndReturnPath (video: VideoModel, remoteStaticPath: string, destPath: string) {
+ return new Promise<string>((res, rej) => {
+ const req = fetchRemoteVideoStaticFile(video, remoteStaticPath, rej)
+
+ const stream = createWriteStream(destPath)
+
+ req.pipe(stream)
+ .on('error', (err) => rej(err))
+ .on('finish', () => res(destPath))
+ })
+ }
+}
diff --git a/server/lib/files-cache/actor-follow-score-cache.ts b/server/lib/files-cache/actor-follow-score-cache.ts
new file mode 100644
index 000000000..d070bde09
--- /dev/null
+++ b/server/lib/files-cache/actor-follow-score-cache.ts
@@ -0,0 +1,46 @@
+import { ACTOR_FOLLOW_SCORE } from '../../initializers'
+import { logger } from '../../helpers/logger'
+
+// Cache follows scores, instead of writing them too often in database
+// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores
+class ActorFollowScoreCache {
+
+ private static instance: ActorFollowScoreCache
+ private pendingFollowsScore: { [ url: string ]: number } = {}
+
+ private constructor () {}
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ updateActorFollowsScore (goodInboxes: string[], badInboxes: string[]) {
+ if (goodInboxes.length === 0 && badInboxes.length === 0) return
+
+ logger.info('Updating %d good actor follows and %d bad actor follows scores in cache.', goodInboxes.length, badInboxes.length)
+
+ for (const goodInbox of goodInboxes) {
+ if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0
+
+ this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS
+ }
+
+ for (const badInbox of badInboxes) {
+ if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0
+
+ this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY
+ }
+ }
+
+ getPendingFollowsScoreCopy () {
+ return this.pendingFollowsScore
+ }
+
+ clearPendingFollowsScore () {
+ this.pendingFollowsScore = {}
+ }
+}
+
+export {
+ ActorFollowScoreCache
+}
diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts
new file mode 100644
index 000000000..e921d04a7
--- /dev/null
+++ b/server/lib/files-cache/index.ts
@@ -0,0 +1,3 @@
+export * from './actor-follow-score-cache'
+export * from './videos-preview-cache'
+export * from './videos-caption-cache'
diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts
new file mode 100644
index 000000000..fe5b441af
--- /dev/null
+++ b/server/lib/files-cache/videos-caption-cache.ts
@@ -0,0 +1,53 @@
+import { join } from 'path'
+import { FILES_CACHE, CONFIG } from '../../initializers'
+import { VideoModel } from '../../models/video/video'
+import { VideoCaptionModel } from '../../models/video/video-caption'
+import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+
+type GetPathParam = { videoId: string, language: string }
+
+class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
+
+ private static readonly KEY_DELIMITER = '%'
+ private static instance: VideosCaptionCache
+
+ private constructor () {
+ super()
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ async getFilePath (params: GetPathParam) {
+ const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language)
+ if (!videoCaption) return undefined
+
+ if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName())
+
+ const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language
+ return this.loadFromLRU(key)
+ }
+
+ protected async loadRemoteFile (key: string) {
+ const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER)
+
+ const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language)
+ if (!videoCaption) return undefined
+
+ if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.')
+
+ // Used to fetch the path
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
+ if (!video) return undefined
+
+ const remoteStaticPath = videoCaption.getCaptionStaticPath()
+ const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
+
+ return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
+ }
+}
+
+export {
+ VideosCaptionCache
+}
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts
new file mode 100644
index 000000000..01cd3647e
--- /dev/null
+++ b/server/lib/files-cache/videos-preview-cache.ts
@@ -0,0 +1,42 @@
+import { join } from 'path'
+import { FILES_CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
+import { VideoModel } from '../../models/video/video'
+import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+
+class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
+
+ private static instance: VideosPreviewCache
+
+ private constructor () {
+ super()
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ async getFilePath (videoUUID: string) {
+ const video = await VideoModel.loadByUUIDWithFile(videoUUID)
+ if (!video) return undefined
+
+ if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
+
+ return this.loadFromLRU(videoUUID)
+ }
+
+ protected async loadRemoteFile (key: string) {
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key)
+ if (!video) return undefined
+
+ if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
+
+ const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
+ const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
+
+ return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
+ }
+}
+
+export {
+ VideosPreviewCache
+}
--
cgit v1.2.3