Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

3 files changed, 0 insertions, 164 deletions

diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
deleted file mode 100644
index f990e9872..000000000
--- a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
+++ /dev/null
@@ -1,132 +0,0 @@
-import express from 'express'
-import { LRUCache } from 'lru-cache'
-import { Model } from 'sequelize'
-import { logger } from '@server/helpers/logger'
-import { CachePromise } from '@server/helpers/promise-cache'
-import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants'
-import { downloadImageFromWorker } from '@server/lib/worker/parent-process'
-import { HttpStatusCode } from '@shared/models'
-
-type ImageModel = {
-  fileUrl: string
-  filename: string
-  onDisk: boolean
-
-  isOwned (): boolean
-  getPath (): string
-
-  save (): Promise<Model>
-}
-
-export abstract class AbstractPermanentFileCache <M extends ImageModel> {
-  // Unsafe because it can return paths that do not exist anymore
-  private readonly filenameToPathUnsafeCache = new LRUCache<string, string>({
-    max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE
-  })
-
-  protected abstract getImageSize (image: M): { width: number, height: number }
-  protected abstract loadModel (filename: string): Promise<M>
-
-  constructor (private readonly directory: string) {
-
-  }
-
-  async lazyServe (options: {
-    filename: string
-    res: express.Response
-    next: express.NextFunction
-  }) {
-    const { filename, res, next } = options
-
-    if (this.filenameToPathUnsafeCache.has(filename)) {
-      return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
-    }
-
-    const image = await this.lazyLoadIfNeeded(filename)
-    if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end()
-
-    const path = image.getPath()
-    this.filenameToPathUnsafeCache.set(filename, path)
-
-    return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => {
-      if (!err) return
-
-      this.onServeError({ err, image, next, filename })
-    })
-  }
-
-  @CachePromise({
-    keyBuilder: filename => filename
-  })
-  private async lazyLoadIfNeeded (filename: string) {
-    const image = await this.loadModel(filename)
-    if (!image) return undefined
-
-    if (image.onDisk === false) {
-      if (!image.fileUrl) return undefined
-
-      try {
-        await this.downloadRemoteFile(image)
-      } catch (err) {
-        logger.warn('Cannot process remote image %s.', image.fileUrl, { err })
-
-        return undefined
-      }
-    }
-
-    return image
-  }
-
-  async downloadRemoteFile (image: M) {
-    logger.info('Download remote image %s lazily.', image.fileUrl)
-
-    const destination = await this.downloadImage({
-      filename: image.filename,
-      fileUrl: image.fileUrl,
-      size: this.getImageSize(image)
-    })
-
-    image.onDisk = true
-    image.save()
-      .catch(err => logger.error('Cannot save new image disk state.', { err }))
-
-    return destination
-  }
-
-  private onServeError (options: {
-    err: any
-    image: M
-    filename: string
-    next: express.NextFunction
-  }) {
-    const { err, image, filename, next } = options
-
-    // It seems this actor image is not on the disk anymore
-    if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) {
-      logger.error('Cannot lazy serve image %s.', filename, { err })
-
-      this.filenameToPathUnsafeCache.delete(filename)
-
-      image.onDisk = false
-      image.save()
-        .catch(err => logger.error('Cannot save new image disk state.', { err }))
-    }
-
-    return next(err)
-  }
-
-  private downloadImage (options: {
-    fileUrl: string
-    filename: string
-    size: { width: number, height: number }
-  }) {
-    const downloaderOptions = {
-      url: options.fileUrl,
-      destDir: this.directory,
-      destName: options.filename,
-      size: options.size
-    }
-
-    return downloadImageFromWorker(downloaderOptions)
-  }
-}
diff --git a/server/lib/files-cache/shared/abstract-simple-file-cache.ts b/server/lib/files-cache/shared/abstract-simple-file-cache.ts
deleted file mode 100644
index 6fab322cd..000000000
--- a/server/lib/files-cache/shared/abstract-simple-file-cache.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-import { remove } from 'fs-extra'
-import { logger } from '../../../helpers/logger'
-import memoizee from 'memoizee'
-
-type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined
-
-export abstract class AbstractSimpleFileCache <T> {
-
-  getFilePath: (params: T) => Promise<GetFilePathResult>
-
-  abstract getFilePathImpl (params: T): Promise<GetFilePathResult>
-
-  // Load and save the remote file, then return the local path from filesystem
-  protected abstract loadRemoteFile (key: string): Promise<GetFilePathResult>
-
-  init (max: number, maxAge: number) {
-    this.getFilePath = memoizee(this.getFilePathImpl, {
-      maxAge,
-      max,
-      promise: true,
-      dispose: (result?: GetFilePathResult) => {
-        if (result && result.isOwned !== true) {
-          remove(result.path)
-            .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name))
-            .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err }))
-        }
-      }
-    })
-  }
-}
diff --git a/server/lib/files-cache/shared/index.ts b/server/lib/files-cache/shared/index.ts
deleted file mode 100644
index 61c4aacc7..000000000
--- a/server/lib/files-cache/shared/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export * from './abstract-permanent-file-cache'
-export * from './abstract-simple-file-cache'