Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

4 files changed, 0 insertions, 354 deletions

diff --git a/server/middlewares/cache/cache.ts b/server/middlewares/cache/cache.ts
deleted file mode 100644
index 6041c76c3..000000000
--- a/server/middlewares/cache/cache.ts
+++ /dev/null
@@ -1,38 +0,0 @@
-import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
-import { ApiCache, APICacheOptions } from './shared'
-
-const defaultOptions: APICacheOptions = {
-  excludeStatus: [
-    HttpStatusCode.FORBIDDEN_403,
-    HttpStatusCode.NOT_FOUND_404
-  ]
-}
-
-function cacheRoute (duration: string) {
-  const instance = new ApiCache(defaultOptions)
-
-  return instance.buildMiddleware(duration)
-}
-
-function cacheRouteFactory (options: APICacheOptions) {
-  const instance = new ApiCache({ ...defaultOptions, ...options })
-
-  return { instance, middleware: instance.buildMiddleware.bind(instance) }
-}
-
-// ---------------------------------------------------------------------------
-
-function buildPodcastGroupsCache (options: {
-  channelId: number
-}) {
-  return 'podcast-feed-' + options.channelId
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  cacheRoute,
-  cacheRouteFactory,
-
-  buildPodcastGroupsCache
-}
diff --git a/server/middlewares/cache/index.ts b/server/middlewares/cache/index.ts
deleted file mode 100644
index 79b512828..000000000
--- a/server/middlewares/cache/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from './cache'
diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts
deleted file mode 100644
index b50b7dce4..000000000
--- a/server/middlewares/cache/shared/api-cache.ts
+++ /dev/null
@@ -1,314 +0,0 @@
-// Thanks: https://github.com/kwhitley/apicache
-// We duplicated the library because it is unmaintened and prevent us to upgrade to recent NodeJS versions
-
-import express from 'express'
-import { OutgoingHttpHeaders } from 'http'
-import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils'
-import { logger } from '@server/helpers/logger'
-import { Redis } from '@server/lib/redis'
-import { asyncMiddleware } from '@server/middlewares'
-import { HttpStatusCode } from '@shared/models'
-
-export interface APICacheOptions {
-  headerBlacklist?: string[]
-  excludeStatus?: HttpStatusCode[]
-}
-
-interface CacheObject {
-  status: number
-  headers: OutgoingHttpHeaders
-  data: any
-  encoding: BufferEncoding
-  timestamp: number
-}
-
-export class ApiCache {
-
-  private readonly options: APICacheOptions
-  private readonly timers: { [ id: string ]: NodeJS.Timeout } = {}
-
-  private readonly index = {
-    groups: [] as string[],
-    all: [] as string[]
-  }
-
-  // Cache keys per group
-  private groups: { [groupIndex: string]: string[] } = {}
-
-  private readonly seed: number
-
-  constructor (options: APICacheOptions) {
-    this.seed = new Date().getTime()
-
-    this.options = {
-      headerBlacklist: [],
-      excludeStatus: [],
-
-      ...options
-    }
-  }
-
-  buildMiddleware (strDuration: string) {
-    const duration = parseDurationToMs(strDuration)
-
-    return asyncMiddleware(
-      async (req: express.Request, res: express.Response, next: express.NextFunction) => {
-        const key = this.getCacheKey(req)
-        const redis = Redis.Instance.getClient()
-
-        if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration)
-
-        try {
-          const obj = await redis.hgetall(key)
-          if (obj?.response) {
-            return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration)
-          }
-
-          return this.makeResponseCacheable(res, next, key, duration)
-        } catch (err) {
-          return this.makeResponseCacheable(res, next, key, duration)
-        }
-      }
-    )
-  }
-
-  clearGroupSafe (group: string) {
-    const run = async () => {
-      const cacheKeys = this.groups[group]
-      if (!cacheKeys) return
-
-      for (const key of cacheKeys) {
-        try {
-          await this.clear(key)
-        } catch (err) {
-          logger.error('Cannot clear ' + key, { err })
-        }
-      }
-
-      delete this.groups[group]
-    }
-
-    void run()
-  }
-
-  private getCacheKey (req: express.Request) {
-    return Redis.Instance.getPrefix() + 'api-cache-' + this.seed + '-' + req.originalUrl
-  }
-
-  private shouldCacheResponse (response: express.Response) {
-    if (!response) return false
-    if (this.options.excludeStatus.includes(response.statusCode)) return false
-
-    return true
-  }
-
-  private addIndexEntries (key: string, res: express.Response) {
-    this.index.all.unshift(key)
-
-    const groups = res.locals.apicacheGroups || []
-
-    for (const group of groups) {
-      if (!this.groups[group]) this.groups[group] = []
-
-      this.groups[group].push(key)
-    }
-  }
-
-  private filterBlacklistedHeaders (headers: OutgoingHttpHeaders) {
-    return Object.keys(headers)
-      .filter(key => !this.options.headerBlacklist.includes(key))
-      .reduce((acc, header) => {
-        acc[header] = headers[header]
-
-        return acc
-      }, {})
-  }
-
-  private createCacheObject (status: number, headers: OutgoingHttpHeaders, data: any, encoding: BufferEncoding) {
-    return {
-      status,
-      headers: this.filterBlacklistedHeaders(headers),
-      data,
-      encoding,
-
-      // Seconds since epoch, used to properly decrement max-age headers in cached responses.
-      timestamp: new Date().getTime() / 1000
-    } as CacheObject
-  }
-
-  private async cacheResponse (key: string, value: object, duration: number) {
-    const redis = Redis.Instance.getClient()
-
-    if (Redis.Instance.isConnected()) {
-      await Promise.all([
-        redis.hset(key, 'response', JSON.stringify(value)),
-        redis.hset(key, 'duration', duration + ''),
-        redis.expire(key, duration / 1000)
-      ])
-    }
-
-    // add automatic cache clearing from duration, includes max limit on setTimeout
-    this.timers[key] = setTimeout(() => {
-      this.clear(key)
-        .catch(err => logger.error('Cannot clear Redis key %s.', key, { err }))
-    }, Math.min(duration, 2147483647))
-  }
-
-  private accumulateContent (res: express.Response, content: any) {
-    if (!content) return
-
-    if (typeof content === 'string') {
-      res.locals.apicache.content = (res.locals.apicache.content || '') + content
-      return
-    }
-
-    if (Buffer.isBuffer(content)) {
-      let oldContent = res.locals.apicache.content
-
-      if (typeof oldContent === 'string') {
-        oldContent = Buffer.from(oldContent)
-      }
-
-      if (!oldContent) {
-        oldContent = Buffer.alloc(0)
-      }
-
-      res.locals.apicache.content = Buffer.concat(
-        [ oldContent, content ],
-        oldContent.length + content.length
-      )
-
-      return
-    }
-
-    res.locals.apicache.content = content
-  }
-
-  private makeResponseCacheable (res: express.Response, next: express.NextFunction, key: string, duration: number) {
-    const self = this
-
-    res.locals.apicache = {
-      write: res.write,
-      writeHead: res.writeHead,
-      end: res.end,
-      cacheable: true,
-      content: undefined,
-      headers: undefined
-    }
-
-    // Patch express
-    res.writeHead = function () {
-      if (self.shouldCacheResponse(res)) {
-        res.setHeader('cache-control', 'max-age=' + (duration / 1000).toFixed(0))
-      } else {
-        res.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
-      }
-
-      res.locals.apicache.headers = Object.assign({}, res.getHeaders())
-      return res.locals.apicache.writeHead.apply(this, arguments as any)
-    }
-
-    res.write = function (chunk: any) {
-      self.accumulateContent(res, chunk)
-      return res.locals.apicache.write.apply(this, arguments as any)
-    }
-
-    res.end = function (content: any, encoding: BufferEncoding) {
-      if (self.shouldCacheResponse(res)) {
-        self.accumulateContent(res, content)
-
-        if (res.locals.apicache.cacheable && res.locals.apicache.content) {
-          self.addIndexEntries(key, res)
-
-          const headers = res.locals.apicache.headers || res.getHeaders()
-          const cacheObject = self.createCacheObject(
-            res.statusCode,
-            headers,
-            res.locals.apicache.content,
-            encoding
-          )
-          self.cacheResponse(key, cacheObject, duration)
-            .catch(err => logger.error('Cannot cache response', { err }))
-        }
-      }
-
-      res.locals.apicache.end.apply(this, arguments as any)
-    } as any
-
-    next()
-  }
-
-  private sendCachedResponse (request: express.Request, response: express.Response, cacheObject: CacheObject, duration: number) {
-    const headers = response.getHeaders()
-
-    if (isTestInstance()) {
-      Object.assign(headers, {
-        'x-api-cache-cached': 'true'
-      })
-    }
-
-    Object.assign(headers, this.filterBlacklistedHeaders(cacheObject.headers || {}), {
-      // Set properly decremented max-age header
-      // This ensures that max-age is in sync with the cache expiration
-      'cache-control':
-        'max-age=' +
-        Math.max(
-          0,
-          (duration / 1000 - (new Date().getTime() / 1000 - cacheObject.timestamp))
-        ).toFixed(0)
-    })
-
-    // unstringify buffers
-    let data = cacheObject.data
-    if (data && data.type === 'Buffer') {
-      data = typeof data.data === 'number'
-        ? Buffer.alloc(data.data)
-        : Buffer.from(data.data)
-    }
-
-    // Test Etag against If-None-Match for 304
-    const cachedEtag = cacheObject.headers.etag
-    const requestEtag = request.headers['if-none-match']
-
-    if (requestEtag && cachedEtag === requestEtag) {
-      response.writeHead(304, headers)
-      return response.end()
-    }
-
-    response.writeHead(cacheObject.status || 200, headers)
-
-    return response.end(data, cacheObject.encoding)
-  }
-
-  private async clear (target: string) {
-    const redis = Redis.Instance.getClient()
-
-    if (target) {
-      clearTimeout(this.timers[target])
-      delete this.timers[target]
-
-      try {
-        await redis.del(target)
-      } catch (err) {
-        logger.error('Cannot delete %s in redis cache.', target, { err })
-      }
-
-      this.index.all = this.index.all.filter(key => key !== target)
-    } else {
-      for (const key of this.index.all) {
-        clearTimeout(this.timers[key])
-        delete this.timers[key]
-
-        try {
-          await redis.del(key)
-        } catch (err) {
-          logger.error('Cannot delete %s in redis cache.', key, { err })
-        }
-      }
-
-      this.index.all = []
-    }
-
-    return this.index
-  }
-}
diff --git a/server/middlewares/cache/shared/index.ts b/server/middlewares/cache/shared/index.ts
deleted file mode 100644
index c707eaf7a..000000000
--- a/server/middlewares/cache/shared/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from './api-cache'