From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- server/middlewares/cache/cache.ts | 38 ---- server/middlewares/cache/index.ts | 1 - server/middlewares/cache/shared/api-cache.ts | 314 --------------------------- server/middlewares/cache/shared/index.ts | 1 - 4 files changed, 354 deletions(-) delete mode 100644 server/middlewares/cache/cache.ts delete mode 100644 server/middlewares/cache/index.ts delete mode 100644 server/middlewares/cache/shared/api-cache.ts delete mode 100644 server/middlewares/cache/shared/index.ts (limited to 'server/middlewares/cache') diff --git a/server/middlewares/cache/cache.ts b/server/middlewares/cache/cache.ts deleted file mode 100644 index 6041c76c3..000000000 --- a/server/middlewares/cache/cache.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { ApiCache, APICacheOptions } from './shared' - -const defaultOptions: APICacheOptions = { - excludeStatus: [ - HttpStatusCode.FORBIDDEN_403, - HttpStatusCode.NOT_FOUND_404 - ] -} - -function cacheRoute (duration: string) { - const instance = new ApiCache(defaultOptions) - - return instance.buildMiddleware(duration) -} - -function cacheRouteFactory (options: APICacheOptions) { - const instance = new ApiCache({ ...defaultOptions, ...options }) - - return { instance, middleware: instance.buildMiddleware.bind(instance) } -} - -// --------------------------------------------------------------------------- - -function buildPodcastGroupsCache (options: { - channelId: number -}) { - return 'podcast-feed-' + options.channelId -} - -// --------------------------------------------------------------------------- - -export { - cacheRoute, - cacheRouteFactory, - - buildPodcastGroupsCache -} diff --git a/server/middlewares/cache/index.ts b/server/middlewares/cache/index.ts deleted file mode 100644 index 79b512828..000000000 --- a/server/middlewares/cache/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './cache' diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts deleted file mode 100644 index b50b7dce4..000000000 --- a/server/middlewares/cache/shared/api-cache.ts +++ /dev/null @@ -1,314 +0,0 @@ -// Thanks: https://github.com/kwhitley/apicache -// We duplicated the library because it is unmaintened and prevent us to upgrade to recent NodeJS versions - -import express from 'express' -import { OutgoingHttpHeaders } from 'http' -import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils' -import { logger } from '@server/helpers/logger' -import { Redis } from '@server/lib/redis' -import { asyncMiddleware } from '@server/middlewares' -import { HttpStatusCode } from '@shared/models' - -export interface APICacheOptions { - headerBlacklist?: string[] - excludeStatus?: HttpStatusCode[] -} - -interface CacheObject { - status: number - headers: OutgoingHttpHeaders - data: any - encoding: BufferEncoding - timestamp: number -} - -export class ApiCache { - - private readonly options: APICacheOptions - private readonly timers: { [ id: string ]: NodeJS.Timeout } = {} - - private readonly index = { - groups: [] as string[], - all: [] as string[] - } - - // Cache keys per group - private groups: { [groupIndex: string]: string[] } = {} - - private readonly seed: number - - constructor (options: APICacheOptions) { - this.seed = new Date().getTime() - - this.options = { - headerBlacklist: [], - excludeStatus: [], - - ...options - } - } - - buildMiddleware (strDuration: string) { - const duration = parseDurationToMs(strDuration) - - return asyncMiddleware( - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const key = this.getCacheKey(req) - const redis = Redis.Instance.getClient() - - if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration) - - try { - const obj = await redis.hgetall(key) - if (obj?.response) { - return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration) - } - - return this.makeResponseCacheable(res, next, key, duration) - } catch (err) { - return this.makeResponseCacheable(res, next, key, duration) - } - } - ) - } - - clearGroupSafe (group: string) { - const run = async () => { - const cacheKeys = this.groups[group] - if (!cacheKeys) return - - for (const key of cacheKeys) { - try { - await this.clear(key) - } catch (err) { - logger.error('Cannot clear ' + key, { err }) - } - } - - delete this.groups[group] - } - - void run() - } - - private getCacheKey (req: express.Request) { - return Redis.Instance.getPrefix() + 'api-cache-' + this.seed + '-' + req.originalUrl - } - - private shouldCacheResponse (response: express.Response) { - if (!response) return false - if (this.options.excludeStatus.includes(response.statusCode)) return false - - return true - } - - private addIndexEntries (key: string, res: express.Response) { - this.index.all.unshift(key) - - const groups = res.locals.apicacheGroups || [] - - for (const group of groups) { - if (!this.groups[group]) this.groups[group] = [] - - this.groups[group].push(key) - } - } - - private filterBlacklistedHeaders (headers: OutgoingHttpHeaders) { - return Object.keys(headers) - .filter(key => !this.options.headerBlacklist.includes(key)) - .reduce((acc, header) => { - acc[header] = headers[header] - - return acc - }, {}) - } - - private createCacheObject (status: number, headers: OutgoingHttpHeaders, data: any, encoding: BufferEncoding) { - return { - status, - headers: this.filterBlacklistedHeaders(headers), - data, - encoding, - - // Seconds since epoch, used to properly decrement max-age headers in cached responses. - timestamp: new Date().getTime() / 1000 - } as CacheObject - } - - private async cacheResponse (key: string, value: object, duration: number) { - const redis = Redis.Instance.getClient() - - if (Redis.Instance.isConnected()) { - await Promise.all([ - redis.hset(key, 'response', JSON.stringify(value)), - redis.hset(key, 'duration', duration + ''), - redis.expire(key, duration / 1000) - ]) - } - - // add automatic cache clearing from duration, includes max limit on setTimeout - this.timers[key] = setTimeout(() => { - this.clear(key) - .catch(err => logger.error('Cannot clear Redis key %s.', key, { err })) - }, Math.min(duration, 2147483647)) - } - - private accumulateContent (res: express.Response, content: any) { - if (!content) return - - if (typeof content === 'string') { - res.locals.apicache.content = (res.locals.apicache.content || '') + content - return - } - - if (Buffer.isBuffer(content)) { - let oldContent = res.locals.apicache.content - - if (typeof oldContent === 'string') { - oldContent = Buffer.from(oldContent) - } - - if (!oldContent) { - oldContent = Buffer.alloc(0) - } - - res.locals.apicache.content = Buffer.concat( - [ oldContent, content ], - oldContent.length + content.length - ) - - return - } - - res.locals.apicache.content = content - } - - private makeResponseCacheable (res: express.Response, next: express.NextFunction, key: string, duration: number) { - const self = this - - res.locals.apicache = { - write: res.write, - writeHead: res.writeHead, - end: res.end, - cacheable: true, - content: undefined, - headers: undefined - } - - // Patch express - res.writeHead = function () { - if (self.shouldCacheResponse(res)) { - res.setHeader('cache-control', 'max-age=' + (duration / 1000).toFixed(0)) - } else { - res.setHeader('cache-control', 'no-cache, no-store, must-revalidate') - } - - res.locals.apicache.headers = Object.assign({}, res.getHeaders()) - return res.locals.apicache.writeHead.apply(this, arguments as any) - } - - res.write = function (chunk: any) { - self.accumulateContent(res, chunk) - return res.locals.apicache.write.apply(this, arguments as any) - } - - res.end = function (content: any, encoding: BufferEncoding) { - if (self.shouldCacheResponse(res)) { - self.accumulateContent(res, content) - - if (res.locals.apicache.cacheable && res.locals.apicache.content) { - self.addIndexEntries(key, res) - - const headers = res.locals.apicache.headers || res.getHeaders() - const cacheObject = self.createCacheObject( - res.statusCode, - headers, - res.locals.apicache.content, - encoding - ) - self.cacheResponse(key, cacheObject, duration) - .catch(err => logger.error('Cannot cache response', { err })) - } - } - - res.locals.apicache.end.apply(this, arguments as any) - } as any - - next() - } - - private sendCachedResponse (request: express.Request, response: express.Response, cacheObject: CacheObject, duration: number) { - const headers = response.getHeaders() - - if (isTestInstance()) { - Object.assign(headers, { - 'x-api-cache-cached': 'true' - }) - } - - Object.assign(headers, this.filterBlacklistedHeaders(cacheObject.headers || {}), { - // Set properly decremented max-age header - // This ensures that max-age is in sync with the cache expiration - 'cache-control': - 'max-age=' + - Math.max( - 0, - (duration / 1000 - (new Date().getTime() / 1000 - cacheObject.timestamp)) - ).toFixed(0) - }) - - // unstringify buffers - let data = cacheObject.data - if (data && data.type === 'Buffer') { - data = typeof data.data === 'number' - ? Buffer.alloc(data.data) - : Buffer.from(data.data) - } - - // Test Etag against If-None-Match for 304 - const cachedEtag = cacheObject.headers.etag - const requestEtag = request.headers['if-none-match'] - - if (requestEtag && cachedEtag === requestEtag) { - response.writeHead(304, headers) - return response.end() - } - - response.writeHead(cacheObject.status || 200, headers) - - return response.end(data, cacheObject.encoding) - } - - private async clear (target: string) { - const redis = Redis.Instance.getClient() - - if (target) { - clearTimeout(this.timers[target]) - delete this.timers[target] - - try { - await redis.del(target) - } catch (err) { - logger.error('Cannot delete %s in redis cache.', target, { err }) - } - - this.index.all = this.index.all.filter(key => key !== target) - } else { - for (const key of this.index.all) { - clearTimeout(this.timers[key]) - delete this.timers[key] - - try { - await redis.del(key) - } catch (err) { - logger.error('Cannot delete %s in redis cache.', key, { err }) - } - } - - this.index.all = [] - } - - return this.index - } -} diff --git a/server/middlewares/cache/shared/index.ts b/server/middlewares/cache/shared/index.ts deleted file mode 100644 index c707eaf7a..000000000 --- a/server/middlewares/cache/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './api-cache' -- cgit v1.2.3