From f162d32da098aa55f6de2367142faa166edb7c08 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 6 Jun 2023 15:59:51 +0200 Subject: Support lazy download thumbnails --- .../shared/abstract-permanent-file-cache.ts | 119 +++++++++++++++++++++ .../shared/abstract-simple-file-cache.ts | 30 ++++++ server/lib/files-cache/shared/index.ts | 2 + 3 files changed, 151 insertions(+) create mode 100644 server/lib/files-cache/shared/abstract-permanent-file-cache.ts create mode 100644 server/lib/files-cache/shared/abstract-simple-file-cache.ts create mode 100644 server/lib/files-cache/shared/index.ts (limited to 'server/lib/files-cache/shared') diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts new file mode 100644 index 000000000..22596c3eb --- /dev/null +++ b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts @@ -0,0 +1,119 @@ +import express from 'express' +import { LRUCache } from 'lru-cache' +import { logger } from '@server/helpers/logger' +import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants' +import { downloadImageFromWorker } from '@server/lib/worker/parent-process' +import { HttpStatusCode } from '@shared/models' +import { Model } from 'sequelize' + +type ImageModel = { + fileUrl: string + filename: string + onDisk: boolean + + isOwned (): boolean + getPath (): string + + save (): Promise +} + +export abstract class AbstractPermanentFileCache { + // Unsafe because it can return paths that do not exist anymore + private readonly filenameToPathUnsafeCache = new LRUCache({ + max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE + }) + + protected abstract getImageSize (image: M): { width: number, height: number } + protected abstract loadModel (filename: string): Promise + + constructor (private readonly directory: string) { + + } + + async lazyServe (options: { + filename: string + res: express.Response + next: express.NextFunction + }) { + const { filename, res, next } = options + + if (this.filenameToPathUnsafeCache.has(filename)) { + return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER }) + } + + const image = await this.loadModel(filename) + if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + if (image.onDisk === false) { + if (!image.fileUrl) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + try { + await this.downloadRemoteFile(image) + } catch (err) { + logger.warn('Cannot process remote image %s.', image.fileUrl, { err }) + + return res.status(HttpStatusCode.NOT_FOUND_404).end() + } + } + + const path = image.getPath() + this.filenameToPathUnsafeCache.set(filename, path) + + return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => { + if (!err) return + + this.onServeError({ err, image, next, filename }) + }) + } + + private async downloadRemoteFile (image: M) { + logger.info('Download remote image %s lazily.', image.fileUrl) + + await this.downloadImage({ + filename: image.filename, + fileUrl: image.fileUrl, + size: this.getImageSize(image) + }) + + image.onDisk = true + image.save() + .catch(err => logger.error('Cannot save new image disk state.', { err })) + } + + private onServeError (options: { + err: any + image: M + filename: string + next: express.NextFunction + }) { + const { err, image, filename, next } = options + + // It seems this actor image is not on the disk anymore + if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) { + logger.error('Cannot lazy serve image %s.', filename, { err }) + + this.filenameToPathUnsafeCache.delete(filename) + + image.onDisk = false + image.save() + .catch(err => logger.error('Cannot save new image disk state.', { err })) + } + + return next(err) + } + + private downloadImage (options: { + fileUrl: string + filename: string + size: { width: number, height: number } + }) { + const downloaderOptions = { + url: options.fileUrl, + destDir: this.directory, + destName: options.filename, + size: options.size + } + + return downloadImageFromWorker(downloaderOptions) + } +} diff --git a/server/lib/files-cache/shared/abstract-simple-file-cache.ts b/server/lib/files-cache/shared/abstract-simple-file-cache.ts new file mode 100644 index 000000000..6fab322cd --- /dev/null +++ b/server/lib/files-cache/shared/abstract-simple-file-cache.ts @@ -0,0 +1,30 @@ +import { remove } from 'fs-extra' +import { logger } from '../../../helpers/logger' +import memoizee from 'memoizee' + +type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined + +export abstract class AbstractSimpleFileCache { + + getFilePath: (params: T) => Promise + + abstract getFilePathImpl (params: T): Promise + + // Load and save the remote file, then return the local path from filesystem + protected abstract loadRemoteFile (key: string): Promise + + init (max: number, maxAge: number) { + this.getFilePath = memoizee(this.getFilePathImpl, { + maxAge, + max, + promise: true, + dispose: (result?: GetFilePathResult) => { + if (result && result.isOwned !== true) { + remove(result.path) + .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name)) + .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err })) + } + } + }) + } +} diff --git a/server/lib/files-cache/shared/index.ts b/server/lib/files-cache/shared/index.ts new file mode 100644 index 000000000..61c4aacc7 --- /dev/null +++ b/server/lib/files-cache/shared/index.ts @@ -0,0 +1,2 @@ +export * from './abstract-permanent-file-cache' +export * from './abstract-simple-file-cache' -- cgit v1.2.3