From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- .../shared/abstract-permanent-file-cache.ts | 132 --------------------- 1 file changed, 132 deletions(-) delete mode 100644 server/lib/files-cache/shared/abstract-permanent-file-cache.ts (limited to 'server/lib/files-cache/shared/abstract-permanent-file-cache.ts') diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts deleted file mode 100644 index f990e9872..000000000 --- a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts +++ /dev/null @@ -1,132 +0,0 @@ -import express from 'express' -import { LRUCache } from 'lru-cache' -import { Model } from 'sequelize' -import { logger } from '@server/helpers/logger' -import { CachePromise } from '@server/helpers/promise-cache' -import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants' -import { downloadImageFromWorker } from '@server/lib/worker/parent-process' -import { HttpStatusCode } from '@shared/models' - -type ImageModel = { - fileUrl: string - filename: string - onDisk: boolean - - isOwned (): boolean - getPath (): string - - save (): Promise -} - -export abstract class AbstractPermanentFileCache { - // Unsafe because it can return paths that do not exist anymore - private readonly filenameToPathUnsafeCache = new LRUCache({ - max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE - }) - - protected abstract getImageSize (image: M): { width: number, height: number } - protected abstract loadModel (filename: string): Promise - - constructor (private readonly directory: string) { - - } - - async lazyServe (options: { - filename: string - res: express.Response - next: express.NextFunction - }) { - const { filename, res, next } = options - - if (this.filenameToPathUnsafeCache.has(filename)) { - return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER }) - } - - const image = await this.lazyLoadIfNeeded(filename) - if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - const path = image.getPath() - this.filenameToPathUnsafeCache.set(filename, path) - - return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => { - if (!err) return - - this.onServeError({ err, image, next, filename }) - }) - } - - @CachePromise({ - keyBuilder: filename => filename - }) - private async lazyLoadIfNeeded (filename: string) { - const image = await this.loadModel(filename) - if (!image) return undefined - - if (image.onDisk === false) { - if (!image.fileUrl) return undefined - - try { - await this.downloadRemoteFile(image) - } catch (err) { - logger.warn('Cannot process remote image %s.', image.fileUrl, { err }) - - return undefined - } - } - - return image - } - - async downloadRemoteFile (image: M) { - logger.info('Download remote image %s lazily.', image.fileUrl) - - const destination = await this.downloadImage({ - filename: image.filename, - fileUrl: image.fileUrl, - size: this.getImageSize(image) - }) - - image.onDisk = true - image.save() - .catch(err => logger.error('Cannot save new image disk state.', { err })) - - return destination - } - - private onServeError (options: { - err: any - image: M - filename: string - next: express.NextFunction - }) { - const { err, image, filename, next } = options - - // It seems this actor image is not on the disk anymore - if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) { - logger.error('Cannot lazy serve image %s.', filename, { err }) - - this.filenameToPathUnsafeCache.delete(filename) - - image.onDisk = false - image.save() - .catch(err => logger.error('Cannot save new image disk state.', { err })) - } - - return next(err) - } - - private downloadImage (options: { - fileUrl: string - filename: string - size: { width: number, height: number } - }) { - const downloaderOptions = { - url: options.fileUrl, - destDir: this.directory, - destName: options.filename, - size: options.size - } - - return downloadImageFromWorker(downloaderOptions) - } -} -- cgit v1.2.3