Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

4 files changed, 0 insertions, 409 deletions

diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts
deleted file mode 100644
index d5cb074df..000000000
--- a/server/lib/object-storage/shared/client.ts
+++ /dev/null
@@ -1,71 +0,0 @@
-import { S3Client } from '@aws-sdk/client-s3'
-import { NodeHttpHandler } from '@aws-sdk/node-http-handler'
-import { logger } from '@server/helpers/logger'
-import { isProxyEnabled } from '@server/helpers/proxy'
-import { getAgent } from '@server/helpers/requests'
-import { CONFIG } from '@server/initializers/config'
-import { lTags } from './logger'
-
-function getProxyRequestHandler () {
-  if (!isProxyEnabled()) return null
-
-  const { agent } = getAgent()
-
-  return new NodeHttpHandler({
-    httpAgent: agent.http,
-    httpsAgent: agent.https
-  })
-}
-
-let endpointParsed: URL
-function getEndpointParsed () {
-  if (endpointParsed) return endpointParsed
-
-  endpointParsed = new URL(getEndpoint())
-
-  return endpointParsed
-}
-
-let s3Client: S3Client
-function getClient () {
-  if (s3Client) return s3Client
-
-  const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE
-
-  s3Client = new S3Client({
-    endpoint: getEndpoint(),
-    region: OBJECT_STORAGE.REGION,
-    credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID
-      ? {
-        accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
-        secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
-      }
-      : undefined,
-    requestHandler: getProxyRequestHandler()
-  })
-
-  logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
-
-  return s3Client
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  getEndpointParsed,
-  getClient
-}
-
-// ---------------------------------------------------------------------------
-
-let endpoint: string
-function getEndpoint () {
-  if (endpoint) return endpoint
-
-  const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT
-  endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://')
-    ? CONFIG.OBJECT_STORAGE.ENDPOINT
-    : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT
-
-  return endpoint
-}
diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts
deleted file mode 100644
index 11e10aa9f..000000000
--- a/server/lib/object-storage/shared/index.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-export * from './client'
-export * from './logger'
-export * from './object-storage-helpers'
diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts
deleted file mode 100644
index 8ab7cbd71..000000000
--- a/server/lib/object-storage/shared/logger.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { loggerTagsFactory } from '@server/helpers/logger'
-
-const lTags = loggerTagsFactory('object-storage')
-
-export {
-  lTags
-}
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
deleted file mode 100644
index 0d8878bd2..000000000
--- a/server/lib/object-storage/shared/object-storage-helpers.ts
+++ /dev/null
@@ -1,328 +0,0 @@
-import { map } from 'bluebird'
-import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra'
-import { dirname } from 'path'
-import { Readable } from 'stream'
-import {
-  _Object,
-  CompleteMultipartUploadCommandOutput,
-  DeleteObjectCommand,
-  GetObjectCommand,
-  ListObjectsV2Command,
-  PutObjectAclCommand,
-  PutObjectCommandInput,
-  S3Client
-} from '@aws-sdk/client-s3'
-import { Upload } from '@aws-sdk/lib-storage'
-import { pipelinePromise } from '@server/helpers/core-utils'
-import { isArray } from '@server/helpers/custom-validators/misc'
-import { logger } from '@server/helpers/logger'
-import { CONFIG } from '@server/initializers/config'
-import { getInternalUrl } from '../urls'
-import { getClient } from './client'
-import { lTags } from './logger'
-
-type BucketInfo = {
-  BUCKET_NAME: string
-  PREFIX?: string
-}
-
-async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
-  const s3Client = getClient()
-
-  const commandPrefix = bucketInfo.PREFIX + prefix
-  const listCommand = new ListObjectsV2Command({
-    Bucket: bucketInfo.BUCKET_NAME,
-    Prefix: commandPrefix
-  })
-
-  const listedObjects = await s3Client.send(listCommand)
-
-  if (isArray(listedObjects.Contents) !== true) return []
-
-  return listedObjects.Contents.map(c => c.Key)
-}
-
-// ---------------------------------------------------------------------------
-
-async function storeObject (options: {
-  inputPath: string
-  objectStorageKey: string
-  bucketInfo: BucketInfo
-  isPrivate: boolean
-}): Promise<string> {
-  const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options
-
-  logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
-
-  const fileStream = createReadStream(inputPath)
-
-  return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate })
-}
-
-async function storeContent (options: {
-  content: string
-  inputPath: string
-  objectStorageKey: string
-  bucketInfo: BucketInfo
-  isPrivate: boolean
-}): Promise<string> {
-  const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options
-
-  logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
-
-  return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate })
-}
-
-// ---------------------------------------------------------------------------
-
-async function updateObjectACL (options: {
-  objectStorageKey: string
-  bucketInfo: BucketInfo
-  isPrivate: boolean
-}) {
-  const { objectStorageKey, bucketInfo, isPrivate } = options
-
-  const acl = getACL(isPrivate)
-  if (!acl) return
-
-  const key = buildKey(objectStorageKey, bucketInfo)
-
-  logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
-
-  const command = new PutObjectAclCommand({
-    Bucket: bucketInfo.BUCKET_NAME,
-    Key: key,
-    ACL: acl
-  })
-
-  await getClient().send(command)
-}
-
-function updatePrefixACL (options: {
-  prefix: string
-  bucketInfo: BucketInfo
-  isPrivate: boolean
-}) {
-  const { prefix, bucketInfo, isPrivate } = options
-
-  const acl = getACL(isPrivate)
-  if (!acl) return
-
-  logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
-
-  return applyOnPrefix({
-    prefix,
-    bucketInfo,
-    commandBuilder: obj => {
-      logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
-
-      return new PutObjectAclCommand({
-        Bucket: bucketInfo.BUCKET_NAME,
-        Key: obj.Key,
-        ACL: acl
-      })
-    }
-  })
-}
-
-// ---------------------------------------------------------------------------
-
-function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
-  const key = buildKey(objectStorageKey, bucketInfo)
-
-  return removeObjectByFullKey(key, bucketInfo)
-}
-
-function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
-  logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
-
-  const command = new DeleteObjectCommand({
-    Bucket: bucketInfo.BUCKET_NAME,
-    Key: fullKey
-  })
-
-  return getClient().send(command)
-}
-
-async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
-  logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
-
-  return applyOnPrefix({
-    prefix,
-    bucketInfo,
-    commandBuilder: obj => {
-      logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
-
-      return new DeleteObjectCommand({
-        Bucket: bucketInfo.BUCKET_NAME,
-        Key: obj.Key
-      })
-    }
-  })
-}
-
-// ---------------------------------------------------------------------------
-
-async function makeAvailable (options: {
-  key: string
-  destination: string
-  bucketInfo: BucketInfo
-}) {
-  const { key, destination, bucketInfo } = options
-
-  await ensureDir(dirname(options.destination))
-
-  const command = new GetObjectCommand({
-    Bucket: bucketInfo.BUCKET_NAME,
-    Key: buildKey(key, bucketInfo)
-  })
-  const response = await getClient().send(command)
-
-  const file = createWriteStream(destination)
-  await pipelinePromise(response.Body as Readable, file)
-
-  file.close()
-}
-
-function buildKey (key: string, bucketInfo: BucketInfo) {
-  return bucketInfo.PREFIX + key
-}
-
-// ---------------------------------------------------------------------------
-
-async function createObjectReadStream (options: {
-  key: string
-  bucketInfo: BucketInfo
-  rangeHeader: string
-}) {
-  const { key, bucketInfo, rangeHeader } = options
-
-  const command = new GetObjectCommand({
-    Bucket: bucketInfo.BUCKET_NAME,
-    Key: buildKey(key, bucketInfo),
-    Range: rangeHeader
-  })
-
-  const response = await getClient().send(command)
-
-  return {
-    response,
-    stream: response.Body as Readable
-  }
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  BucketInfo,
-  buildKey,
-
-  storeObject,
-  storeContent,
-
-  removeObject,
-  removeObjectByFullKey,
-  removePrefix,
-
-  makeAvailable,
-
-  updateObjectACL,
-  updatePrefixACL,
-
-  listKeysOfPrefix,
-  createObjectReadStream
-}
-
-// ---------------------------------------------------------------------------
-
-async function uploadToStorage (options: {
-  content: ReadStream | string
-  objectStorageKey: string
-  bucketInfo: BucketInfo
-  isPrivate: boolean
-}) {
-  const { content, objectStorageKey, bucketInfo, isPrivate } = options
-
-  const input: PutObjectCommandInput = {
-    Body: content,
-    Bucket: bucketInfo.BUCKET_NAME,
-    Key: buildKey(objectStorageKey, bucketInfo)
-  }
-
-  const acl = getACL(isPrivate)
-  if (acl) input.ACL = acl
-
-  const parallelUploads3 = new Upload({
-    client: getClient(),
-    queueSize: 4,
-    partSize: CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART,
-
-    // `leavePartsOnError` must be set to `true` to avoid silently dropping failed parts
-    // More detailed explanation:
-    // https://github.com/aws/aws-sdk-js-v3/blob/v3.164.0/lib/lib-storage/src/Upload.ts#L274
-    // https://github.com/aws/aws-sdk-js-v3/issues/2311#issuecomment-939413928
-    leavePartsOnError: true,
-    params: input
-  })
-
-  const response = (await parallelUploads3.done()) as CompleteMultipartUploadCommandOutput
-  // Check is needed even if the HTTP status code is 200 OK
-  // For more information, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
-  if (!response.Bucket) {
-    const message = `Error uploading ${objectStorageKey} to bucket ${bucketInfo.BUCKET_NAME}`
-    logger.error(message, { response, ...lTags() })
-    throw new Error(message)
-  }
-
-  logger.debug(
-    'Completed %s%s in bucket %s',
-    bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, { ...lTags(), reseponseMetadata: response.$metadata }
-  )
-
-  return getInternalUrl(bucketInfo, objectStorageKey)
-}
-
-async function applyOnPrefix (options: {
-  prefix: string
-  bucketInfo: BucketInfo
-  commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0]
-
-  continuationToken?: string
-}) {
-  const { prefix, bucketInfo, commandBuilder, continuationToken } = options
-
-  const s3Client = getClient()
-
-  const commandPrefix = buildKey(prefix, bucketInfo)
-  const listCommand = new ListObjectsV2Command({
-    Bucket: bucketInfo.BUCKET_NAME,
-    Prefix: commandPrefix,
-    ContinuationToken: continuationToken
-  })
-
-  const listedObjects = await s3Client.send(listCommand)
-
-  if (isArray(listedObjects.Contents) !== true) {
-    const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
-
-    logger.error(message, { response: listedObjects, ...lTags() })
-    throw new Error(message)
-  }
-
-  await map(listedObjects.Contents, object => {
-    const command = commandBuilder(object)
-
-    return s3Client.send(command)
-  }, { concurrency: 10 })
-
-  // Repeat if not all objects could be listed at once (limit of 1000?)
-  if (listedObjects.IsTruncated) {
-    await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken })
-  }
-}
-
-function getACL (isPrivate: boolean) {
-  return isPrivate
-    ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE
-    : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC
-}