Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

10 files changed, 405 insertions, 0 deletions

diff --git a/apps/peertube-runner/src/shared/config-manager.ts b/apps/peertube-runner/src/shared/config-manager.ts
new file mode 100644
index 000000000..84a326a16
--- /dev/null
+++ b/apps/peertube-runner/src/shared/config-manager.ts
@@ -0,0 +1,140 @@
+import { parse, stringify } from '@iarna/toml'
+import envPaths from 'env-paths'
+import { ensureDir, pathExists, remove } from 'fs-extra/esm'
+import { readFile, writeFile } from 'fs/promises'
+import merge from 'lodash-es/merge.js'
+import { dirname, join } from 'path'
+import { logger } from '../shared/index.js'
+
+const paths = envPaths('peertube-runner')
+
+type Config = {
+  jobs: {
+    concurrency: number
+  }
+
+  ffmpeg: {
+    threads: number
+    nice: number
+  }
+
+  registeredInstances: {
+    url: string
+    runnerToken: string
+    runnerName: string
+    runnerDescription?: string
+  }[]
+}
+
+export class ConfigManager {
+  private static instance: ConfigManager
+
+  private config: Config = {
+    jobs: {
+      concurrency: 2
+    },
+    ffmpeg: {
+      threads: 2,
+      nice: 20
+    },
+    registeredInstances: []
+  }
+
+  private id: string
+  private configFilePath: string
+
+  private constructor () {}
+
+  init (id: string) {
+    this.id = id
+    this.configFilePath = join(this.getConfigDir(), 'config.toml')
+  }
+
+  async load () {
+    logger.info(`Using ${this.configFilePath} as configuration file`)
+
+    if (this.isTestInstance()) {
+      logger.info('Removing configuration file as we are using the "test" id')
+      await remove(this.configFilePath)
+    }
+
+    await ensureDir(dirname(this.configFilePath))
+
+    if (!await pathExists(this.configFilePath)) {
+      await this.save()
+    }
+
+    const file = await readFile(this.configFilePath, 'utf-8')
+
+    this.config = merge(this.config, parse(file))
+  }
+
+  save () {
+    return writeFile(this.configFilePath, stringify(this.config))
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async setRegisteredInstances (registeredInstances: {
+    url: string
+    runnerToken: string
+    runnerName: string
+    runnerDescription?: string
+  }[]) {
+    this.config.registeredInstances = registeredInstances
+
+    await this.save()
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getConfig () {
+    return this.deepFreeze(this.config)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getTranscodingDirectory () {
+    return join(paths.cache, this.id, 'transcoding')
+  }
+
+  getSocketDirectory () {
+    return join(paths.data, this.id)
+  }
+
+  getSocketPath () {
+    return join(this.getSocketDirectory(), 'peertube-runner.sock')
+  }
+
+  getConfigDir () {
+    return join(paths.config, this.id)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  isTestInstance () {
+    return typeof this.id === 'string' && this.id.match(/^test-\d$/)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/freeze
+  private deepFreeze <T extends object> (object: T) {
+    const propNames = Reflect.ownKeys(object)
+
+    // Freeze properties before freezing self
+    for (const name of propNames) {
+      const value = object[name]
+
+      if ((value && typeof value === 'object') || typeof value === 'function') {
+        this.deepFreeze(value)
+      }
+    }
+
+    return Object.freeze({ ...object })
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}
diff --git a/apps/peertube-runner/src/shared/http.ts b/apps/peertube-runner/src/shared/http.ts
new file mode 100644
index 000000000..42886279c
--- /dev/null
+++ b/apps/peertube-runner/src/shared/http.ts
@@ -0,0 +1,67 @@
+import { createWriteStream } from 'fs'
+import { remove } from 'fs-extra/esm'
+import { request as requestHTTP } from 'http'
+import { request as requestHTTPS, RequestOptions } from 'https'
+import { logger } from './logger.js'
+
+export function downloadFile (options: {
+  url: string
+  destination: string
+  runnerToken: string
+  jobToken: string
+}) {
+  const { url, destination, runnerToken, jobToken } = options
+
+  logger.debug(`Downloading file ${url}`)
+
+  return new Promise<void>((res, rej) => {
+    const parsed = new URL(url)
+
+    const body = JSON.stringify({
+      runnerToken,
+      jobToken
+    })
+
+    const getOptions: RequestOptions = {
+      method: 'POST',
+      hostname: parsed.hostname,
+      port: parsed.port,
+      path: parsed.pathname,
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(body, 'utf-8')
+      }
+    }
+
+    const request = getRequest(url)(getOptions, response => {
+      const code = response.statusCode ?? 0
+
+      if (code >= 400) {
+        return rej(new Error(response.statusMessage))
+      }
+
+      const file = createWriteStream(destination)
+      file.on('finish', () => res())
+
+      response.pipe(file)
+    })
+
+    request.on('error', err => {
+      remove(destination)
+        .catch(err => logger.error(err))
+
+      return rej(err)
+    })
+
+    request.write(body)
+    request.end()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function getRequest (url: string) {
+  if (url.startsWith('https://')) return requestHTTPS
+
+  return requestHTTP
+}
diff --git a/apps/peertube-runner/src/shared/index.ts b/apps/peertube-runner/src/shared/index.ts
new file mode 100644
index 000000000..951eef55b
--- /dev/null
+++ b/apps/peertube-runner/src/shared/index.ts
@@ -0,0 +1,3 @@
+export * from './config-manager.js'
+export * from './http.js'
+export * from './logger.js'
diff --git a/apps/peertube-runner/src/shared/ipc/index.ts b/apps/peertube-runner/src/shared/ipc/index.ts
new file mode 100644
index 000000000..337d4de16
--- /dev/null
+++ b/apps/peertube-runner/src/shared/ipc/index.ts
@@ -0,0 +1,2 @@
+export * from './ipc-client.js'
+export * from './ipc-server.js'
diff --git a/apps/peertube-runner/src/shared/ipc/ipc-client.ts b/apps/peertube-runner/src/shared/ipc/ipc-client.ts
new file mode 100644
index 000000000..aa5740dd1
--- /dev/null
+++ b/apps/peertube-runner/src/shared/ipc/ipc-client.ts
@@ -0,0 +1,88 @@
+import CliTable3 from 'cli-table3'
+import { ensureDir } from 'fs-extra/esm'
+import { Client as NetIPC } from 'net-ipc'
+import { ConfigManager } from '../config-manager.js'
+import { IPCReponse, IPCReponseData, IPCRequest } from './shared/index.js'
+
+export class IPCClient {
+  private netIPC: NetIPC
+
+  async run () {
+    await ensureDir(ConfigManager.Instance.getSocketDirectory())
+
+    const socketPath = ConfigManager.Instance.getSocketPath()
+
+    this.netIPC = new NetIPC({ path: socketPath })
+
+    try {
+      await this.netIPC.connect()
+    } catch (err) {
+      if (err.code === 'ECONNREFUSED') {
+        throw new Error(
+          'This runner is not currently running in server mode on this system. ' +
+          'Please run it using the `server` command first (in another terminal for example) and then retry your command.'
+        )
+      }
+
+      throw err
+    }
+  }
+
+  async askRegister (options: {
+    url: string
+    registrationToken: string
+    runnerName: string
+    runnerDescription?: string
+  }) {
+    const req: IPCRequest = {
+      type: 'register',
+      ...options
+    }
+
+    const { success, error } = await this.netIPC.request(req) as IPCReponse
+
+    if (success) console.log('PeerTube instance registered')
+    else console.error('Could not register PeerTube instance on runner server side', error)
+  }
+
+  async askUnregister (options: {
+    url: string
+    runnerName: string
+  }) {
+    const req: IPCRequest = {
+      type: 'unregister',
+      ...options
+    }
+
+    const { success, error } = await this.netIPC.request(req) as IPCReponse
+
+    if (success) console.log('PeerTube instance unregistered')
+    else console.error('Could not unregister PeerTube instance on runner server side', error)
+  }
+
+  async askListRegistered () {
+    const req: IPCRequest = {
+      type: 'list-registered'
+    }
+
+    const { success, error, data } = await this.netIPC.request(req) as IPCReponse<IPCReponseData>
+    if (!success) {
+      console.error('Could not list registered PeerTube instances', error)
+      return
+    }
+
+    const table = new CliTable3({
+      head: [ 'instance', 'runner name', 'runner description' ]
+    })
+
+    for (const server of data.servers) {
+      table.push([ server.url, server.runnerName, server.runnerDescription ])
+    }
+
+    console.log(table.toString())
+  }
+
+  stop () {
+    this.netIPC.destroy()
+  }
+}
diff --git a/apps/peertube-runner/src/shared/ipc/ipc-server.ts b/apps/peertube-runner/src/shared/ipc/ipc-server.ts
new file mode 100644
index 000000000..c68438504
--- /dev/null
+++ b/apps/peertube-runner/src/shared/ipc/ipc-server.ts
@@ -0,0 +1,61 @@
+import { ensureDir } from 'fs-extra/esm'
+import { Server as NetIPC } from 'net-ipc'
+import { pick } from '@peertube/peertube-core-utils'
+import { RunnerServer } from '../../server/index.js'
+import { ConfigManager } from '../config-manager.js'
+import { logger } from '../logger.js'
+import { IPCReponse, IPCReponseData, IPCRequest } from './shared/index.js'
+
+export class IPCServer {
+  private netIPC: NetIPC
+  private runnerServer: RunnerServer
+
+  async run (runnerServer: RunnerServer) {
+    this.runnerServer = runnerServer
+
+    await ensureDir(ConfigManager.Instance.getSocketDirectory())
+
+    const socketPath = ConfigManager.Instance.getSocketPath()
+    this.netIPC = new NetIPC({ path: socketPath })
+    await this.netIPC.start()
+
+    logger.info(`IPC socket created on ${socketPath}`)
+
+    this.netIPC.on('request', async (req: IPCRequest, res) => {
+      try {
+        const data = await this.process(req)
+
+        this.sendReponse(res, { success: true, data })
+      } catch (err) {
+        logger.error('Cannot execute RPC call', err)
+        this.sendReponse(res, { success: false, error: err.message })
+      }
+    })
+  }
+
+  private async process (req: IPCRequest) {
+    switch (req.type) {
+      case 'register':
+        await this.runnerServer.registerRunner(pick(req, [ 'url', 'registrationToken', 'runnerName', 'runnerDescription' ]))
+        return undefined
+
+      case 'unregister':
+        await this.runnerServer.unregisterRunner(pick(req, [ 'url', 'runnerName' ]))
+        return undefined
+
+      case 'list-registered':
+        return Promise.resolve(this.runnerServer.listRegistered())
+
+      default:
+        throw new Error('Unknown RPC call ' + (req as any).type)
+    }
+  }
+
+  private sendReponse <T extends IPCReponseData> (
+    response: (data: any) => Promise<void>,
+    body: IPCReponse<T>
+  ) {
+    response(body)
+      .catch(err => logger.error('Cannot send response after IPC request', err))
+  }
+}
diff --git a/apps/peertube-runner/src/shared/ipc/shared/index.ts b/apps/peertube-runner/src/shared/ipc/shared/index.ts
new file mode 100644
index 000000000..986acafb0
--- /dev/null
+++ b/apps/peertube-runner/src/shared/ipc/shared/index.ts
@@ -0,0 +1,2 @@
+export * from './ipc-request.model.js'
+export * from './ipc-response.model.js'
diff --git a/apps/peertube-runner/src/shared/ipc/shared/ipc-request.model.ts b/apps/peertube-runner/src/shared/ipc/shared/ipc-request.model.ts
new file mode 100644
index 000000000..352808c74
--- /dev/null
+++ b/apps/peertube-runner/src/shared/ipc/shared/ipc-request.model.ts
@@ -0,0 +1,15 @@
+export type IPCRequest =
+  IPCRequestRegister |
+  IPCRequestUnregister |
+  IPCRequestListRegistered
+
+export type IPCRequestRegister = {
+  type: 'register'
+  url: string
+  registrationToken: string
+  runnerName: string
+  runnerDescription?: string
+}
+
+export type IPCRequestUnregister = { type: 'unregister', url: string, runnerName: string }
+export type IPCRequestListRegistered = { type: 'list-registered' }
diff --git a/apps/peertube-runner/src/shared/ipc/shared/ipc-response.model.ts b/apps/peertube-runner/src/shared/ipc/shared/ipc-response.model.ts
new file mode 100644
index 000000000..689d6e09a
--- /dev/null
+++ b/apps/peertube-runner/src/shared/ipc/shared/ipc-response.model.ts
@@ -0,0 +1,15 @@
+export type IPCReponse <T extends IPCReponseData = undefined> = {
+  success: boolean
+  error?: string
+  data?: T
+}
+
+export type IPCReponseData =
+  // list registered
+  {
+    servers: {
+      runnerName: string
+      runnerDescription: string
+      url: string
+    }[]
+  }
diff --git a/apps/peertube-runner/src/shared/logger.ts b/apps/peertube-runner/src/shared/logger.ts
new file mode 100644
index 000000000..ef5283892
--- /dev/null
+++ b/apps/peertube-runner/src/shared/logger.ts
@@ -0,0 +1,12 @@
+import { pino } from 'pino'
+import pretty from 'pino-pretty'
+
+const logger = pino(pretty.default({
+  colorize: true
+}))
+
+logger.level = 'info'
+
+export {
+  logger
+}