Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

4 files changed, 0 insertions, 430 deletions

diff --git a/shared/server-commands/runners/index.ts b/shared/server-commands/runners/index.ts
deleted file mode 100644
index 9e8e1baf2..000000000
--- a/shared/server-commands/runners/index.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-export * from './runner-jobs-command'
-export * from './runner-registration-tokens-command'
-export * from './runners-command'
diff --git a/shared/server-commands/runners/runner-jobs-command.ts b/shared/server-commands/runners/runner-jobs-command.ts
deleted file mode 100644
index 0a0ffb5d3..000000000
--- a/shared/server-commands/runners/runner-jobs-command.ts
+++ /dev/null
@@ -1,294 +0,0 @@
-import { omit, pick, wait } from '@shared/core-utils'
-import {
-  AbortRunnerJobBody,
-  AcceptRunnerJobBody,
-  AcceptRunnerJobResult,
-  ErrorRunnerJobBody,
-  HttpStatusCode,
-  isHLSTranscodingPayloadSuccess,
-  isLiveRTMPHLSTranscodingUpdatePayload,
-  isWebVideoOrAudioMergeTranscodingPayloadSuccess,
-  ListRunnerJobsQuery,
-  RequestRunnerJobBody,
-  RequestRunnerJobResult,
-  ResultList,
-  RunnerJobAdmin,
-  RunnerJobLiveRTMPHLSTranscodingPayload,
-  RunnerJobPayload,
-  RunnerJobState,
-  RunnerJobSuccessBody,
-  RunnerJobSuccessPayload,
-  RunnerJobType,
-  RunnerJobUpdateBody,
-  RunnerJobVODPayload
-} from '@shared/models'
-import { unwrapBody } from '../requests'
-import { waitJobs } from '../server'
-import { AbstractCommand, OverrideCommandOptions } from '../shared'
-
-export class RunnerJobsCommand extends AbstractCommand {
-
-  list (options: OverrideCommandOptions & ListRunnerJobsQuery = {}) {
-    const path = '/api/v1/runners/jobs'
-
-    return this.getRequestBody<ResultList<RunnerJobAdmin>>({
-      ...options,
-
-      path,
-      query: pick(options, [ 'start', 'count', 'sort', 'search', 'stateOneOf' ]),
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    })
-  }
-
-  cancelByAdmin (options: OverrideCommandOptions & { jobUUID: string }) {
-    const path = '/api/v1/runners/jobs/' + options.jobUUID + '/cancel'
-
-    return this.postBodyRequest({
-      ...options,
-
-      path,
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  deleteByAdmin (options: OverrideCommandOptions & { jobUUID: string }) {
-    const path = '/api/v1/runners/jobs/' + options.jobUUID
-
-    return this.deleteRequest({
-      ...options,
-
-      path,
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  // ---------------------------------------------------------------------------
-
-  request (options: OverrideCommandOptions & RequestRunnerJobBody) {
-    const path = '/api/v1/runners/jobs/request'
-
-    return unwrapBody<RequestRunnerJobResult>(this.postBodyRequest({
-      ...options,
-
-      path,
-      fields: pick(options, [ 'runnerToken' ]),
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    }))
-  }
-
-  async requestVOD (options: OverrideCommandOptions & RequestRunnerJobBody) {
-    const vodTypes = new Set<RunnerJobType>([ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ])
-
-    const { availableJobs } = await this.request(options)
-
-    return {
-      availableJobs: availableJobs.filter(j => vodTypes.has(j.type))
-    } as RequestRunnerJobResult<RunnerJobVODPayload>
-  }
-
-  async requestLive (options: OverrideCommandOptions & RequestRunnerJobBody) {
-    const vodTypes = new Set<RunnerJobType>([ 'live-rtmp-hls-transcoding' ])
-
-    const { availableJobs } = await this.request(options)
-
-    return {
-      availableJobs: availableJobs.filter(j => vodTypes.has(j.type))
-    } as RequestRunnerJobResult<RunnerJobLiveRTMPHLSTranscodingPayload>
-  }
-
-  // ---------------------------------------------------------------------------
-
-  accept <T extends RunnerJobPayload = RunnerJobPayload> (options: OverrideCommandOptions & AcceptRunnerJobBody & { jobUUID: string }) {
-    const path = '/api/v1/runners/jobs/' + options.jobUUID + '/accept'
-
-    return unwrapBody<AcceptRunnerJobResult<T>>(this.postBodyRequest({
-      ...options,
-
-      path,
-      fields: pick(options, [ 'runnerToken' ]),
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    }))
-  }
-
-  abort (options: OverrideCommandOptions & AbortRunnerJobBody & { jobUUID: string }) {
-    const path = '/api/v1/runners/jobs/' + options.jobUUID + '/abort'
-
-    return this.postBodyRequest({
-      ...options,
-
-      path,
-      fields: pick(options, [ 'reason', 'jobToken', 'runnerToken' ]),
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  update (options: OverrideCommandOptions & RunnerJobUpdateBody & { jobUUID: string }) {
-    const path = '/api/v1/runners/jobs/' + options.jobUUID + '/update'
-
-    const { payload } = options
-    const attaches: { [id: string]: any } = {}
-    let payloadWithoutFiles = payload
-
-    if (isLiveRTMPHLSTranscodingUpdatePayload(payload)) {
-      if (payload.masterPlaylistFile) {
-        attaches[`payload[masterPlaylistFile]`] = payload.masterPlaylistFile
-      }
-
-      attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile
-      attaches[`payload[videoChunkFile]`] = payload.videoChunkFile
-
-      payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'masterPlaylistFile', 'resolutionPlaylistFile', 'videoChunkFile' ])
-    }
-
-    return this.postUploadRequest({
-      ...options,
-
-      path,
-      fields: {
-        ...pick(options, [ 'progress', 'jobToken', 'runnerToken' ]),
-
-        payload: payloadWithoutFiles
-      },
-      attaches,
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  error (options: OverrideCommandOptions & ErrorRunnerJobBody & { jobUUID: string }) {
-    const path = '/api/v1/runners/jobs/' + options.jobUUID + '/error'
-
-    return this.postBodyRequest({
-      ...options,
-
-      path,
-      fields: pick(options, [ 'message', 'jobToken', 'runnerToken' ]),
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  success (options: OverrideCommandOptions & RunnerJobSuccessBody & { jobUUID: string }) {
-    const { payload } = options
-
-    const path = '/api/v1/runners/jobs/' + options.jobUUID + '/success'
-    const attaches: { [id: string]: any } = {}
-    let payloadWithoutFiles = payload
-
-    if ((isWebVideoOrAudioMergeTranscodingPayloadSuccess(payload) || isHLSTranscodingPayloadSuccess(payload)) && payload.videoFile) {
-      attaches[`payload[videoFile]`] = payload.videoFile
-
-      payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'videoFile' ])
-    }
-
-    if (isHLSTranscodingPayloadSuccess(payload) && payload.resolutionPlaylistFile) {
-      attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile
-
-      payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'resolutionPlaylistFile' ])
-    }
-
-    return this.postUploadRequest({
-      ...options,
-
-      path,
-      attaches,
-      fields: {
-        ...pick(options, [ 'jobToken', 'runnerToken' ]),
-
-        payload: payloadWithoutFiles
-      },
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  getJobFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) {
-    const { host, protocol, pathname } = new URL(options.url)
-
-    return this.postBodyRequest({
-      url: `${protocol}//${host}`,
-      path: pathname,
-
-      fields: pick(options, [ 'jobToken', 'runnerToken' ]),
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    })
-  }
-
-  // ---------------------------------------------------------------------------
-
-  async autoAccept (options: OverrideCommandOptions & RequestRunnerJobBody & { type?: RunnerJobType }) {
-    const { availableJobs } = await this.request(options)
-
-    const job = options.type
-      ? availableJobs.find(j => j.type === options.type)
-      : availableJobs[0]
-
-    return this.accept({ ...options, jobUUID: job.uuid })
-  }
-
-  async autoProcessWebVideoJob (runnerToken: string, jobUUIDToProcess?: string) {
-    let jobUUID = jobUUIDToProcess
-
-    if (!jobUUID) {
-      const { availableJobs } = await this.request({ runnerToken })
-      jobUUID = availableJobs[0].uuid
-    }
-
-    const { job } = await this.accept({ runnerToken, jobUUID })
-    const jobToken = job.jobToken
-
-    const payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' }
-    await this.success({ runnerToken, jobUUID, jobToken, payload })
-
-    await waitJobs([ this.server ])
-
-    return job
-  }
-
-  async cancelAllJobs (options: { state?: RunnerJobState } = {}) {
-    const { state } = options
-
-    const { data } = await this.list({ count: 100 })
-
-    const allowedStates = new Set<RunnerJobState>([
-      RunnerJobState.PENDING,
-      RunnerJobState.PROCESSING,
-      RunnerJobState.WAITING_FOR_PARENT_JOB
-    ])
-
-    for (const job of data) {
-      if (state && job.state.id !== state) continue
-      else if (allowedStates.has(job.state.id) !== true) continue
-
-      await this.cancelByAdmin({ jobUUID: job.uuid })
-    }
-  }
-
-  async getJob (options: OverrideCommandOptions & { uuid: string }) {
-    const { data } = await this.list({ ...options, count: 100, sort: '-updatedAt' })
-
-    return data.find(j => j.uuid === options.uuid)
-  }
-
-  async requestLiveJob (runnerToken: string) {
-    let availableJobs: RequestRunnerJobResult<RunnerJobLiveRTMPHLSTranscodingPayload>['availableJobs'] = []
-
-    while (availableJobs.length === 0) {
-      const result = await this.requestLive({ runnerToken })
-      availableJobs = result.availableJobs
-
-      if (availableJobs.length === 1) break
-
-      await wait(150)
-    }
-
-    return availableJobs[0]
-  }
-}
diff --git a/shared/server-commands/runners/runner-registration-tokens-command.ts b/shared/server-commands/runners/runner-registration-tokens-command.ts
deleted file mode 100644
index e4f2e3d95..000000000
--- a/shared/server-commands/runners/runner-registration-tokens-command.ts
+++ /dev/null
@@ -1,55 +0,0 @@
-import { pick } from '@shared/core-utils'
-import { HttpStatusCode, ResultList, RunnerRegistrationToken } from '@shared/models'
-import { AbstractCommand, OverrideCommandOptions } from '../shared'
-
-export class RunnerRegistrationTokensCommand extends AbstractCommand {
-
-  list (options: OverrideCommandOptions & {
-    start?: number
-    count?: number
-    sort?: string
-  } = {}) {
-    const path = '/api/v1/runners/registration-tokens'
-
-    return this.getRequestBody<ResultList<RunnerRegistrationToken>>({
-      ...options,
-
-      path,
-      query: pick(options, [ 'start', 'count', 'sort' ]),
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    })
-  }
-
-  generate (options: OverrideCommandOptions = {}) {
-    const path = '/api/v1/runners/registration-tokens/generate'
-
-    return this.postBodyRequest({
-      ...options,
-
-      path,
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  delete (options: OverrideCommandOptions & {
-    id: number
-  }) {
-    const path = '/api/v1/runners/registration-tokens/' + options.id
-
-    return this.deleteRequest({
-      ...options,
-
-      path,
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  async getFirstRegistrationToken (options: OverrideCommandOptions = {}) {
-    const { data } = await this.list(options)
-
-    return data[0].registrationToken
-  }
-}
diff --git a/shared/server-commands/runners/runners-command.ts b/shared/server-commands/runners/runners-command.ts
deleted file mode 100644
index b0083e841..000000000
--- a/shared/server-commands/runners/runners-command.ts
+++ /dev/null
@@ -1,78 +0,0 @@
-import { pick } from '@shared/core-utils'
-import { buildUUID } from '@shared/extra-utils'
-import { HttpStatusCode, RegisterRunnerBody, RegisterRunnerResult, ResultList, Runner, UnregisterRunnerBody } from '@shared/models'
-import { unwrapBody } from '../requests'
-import { AbstractCommand, OverrideCommandOptions } from '../shared'
-
-export class RunnersCommand extends AbstractCommand {
-
-  list (options: OverrideCommandOptions & {
-    start?: number
-    count?: number
-    sort?: string
-  } = {}) {
-    const path = '/api/v1/runners'
-
-    return this.getRequestBody<ResultList<Runner>>({
-      ...options,
-
-      path,
-      query: pick(options, [ 'start', 'count', 'sort' ]),
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    })
-  }
-
-  register (options: OverrideCommandOptions & RegisterRunnerBody) {
-    const path = '/api/v1/runners/register'
-
-    return unwrapBody<RegisterRunnerResult>(this.postBodyRequest({
-      ...options,
-
-      path,
-      fields: pick(options, [ 'name', 'registrationToken', 'description' ]),
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    }))
-  }
-
-  unregister (options: OverrideCommandOptions & UnregisterRunnerBody) {
-    const path = '/api/v1/runners/unregister'
-
-    return this.postBodyRequest({
-      ...options,
-
-      path,
-      fields: pick(options, [ 'runnerToken' ]),
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  delete (options: OverrideCommandOptions & {
-    id: number
-  }) {
-    const path = '/api/v1/runners/' + options.id
-
-    return this.deleteRequest({
-      ...options,
-
-      path,
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
-  // ---------------------------------------------------------------------------
-
-  async autoRegisterRunner () {
-    const { data } = await this.server.runnerRegistrationTokens.list({ sort: 'createdAt' })
-
-    const { runnerToken } = await this.register({
-      name: 'runner ' + buildUUID(),
-      registrationToken: data[0].registrationToken
-    })
-
-    return runnerToken
-  }
-}