Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

1 files changed, 0 insertions, 330 deletions

diff --git a/server/tests/api/runners/runner-live-transcoding.ts b/server/tests/api/runners/runner-live-transcoding.ts
deleted file mode 100644
index b11d54039..000000000
--- a/server/tests/api/runners/runner-live-transcoding.ts
+++ /dev/null
@@ -1,330 +0,0 @@
-/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
-
-import { expect } from 'chai'
-import { FfmpegCommand } from 'fluent-ffmpeg'
-import { readFile } from 'fs-extra'
-import { buildAbsoluteFixturePath, wait } from '@shared/core-utils'
-import {
-  HttpStatusCode,
-  LiveRTMPHLSTranscodingUpdatePayload,
-  LiveVideo,
-  LiveVideoError,
-  RunnerJob,
-  RunnerJobLiveRTMPHLSTranscodingPayload,
-  Video,
-  VideoPrivacy,
-  VideoState
-} from '@shared/models'
-import {
-  cleanupTests,
-  createSingleServer,
-  makeRawRequest,
-  PeerTubeServer,
-  sendRTMPStream,
-  setAccessTokensToServers,
-  setDefaultVideoChannel,
-  stopFfmpeg,
-  testFfmpegStreamError,
-  waitJobs
-} from '@shared/server-commands'
-
-describe('Test runner live transcoding', function () {
-  let server: PeerTubeServer
-  let runnerToken: string
-  let baseUrl: string
-
-  before(async function () {
-    this.timeout(120_000)
-
-    server = await createSingleServer(1)
-
-    await setAccessTokensToServers([ server ])
-    await setDefaultVideoChannel([ server ])
-
-    await server.config.enableRemoteTranscoding()
-    await server.config.enableTranscoding()
-    runnerToken = await server.runners.autoRegisterRunner()
-
-    baseUrl = server.url + '/static/streaming-playlists/hls'
-  })
-
-  describe('Without transcoding enabled', function () {
-
-    before(async function () {
-      await server.config.enableLive({
-        allowReplay: false,
-        resolutions: 'min',
-        transcoding: false
-      })
-    })
-
-    it('Should not have available jobs', async function () {
-      this.timeout(120000)
-
-      const { live, video } = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC })
-
-      const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
-      await server.live.waitUntilPublished({ videoId: video.id })
-
-      await waitJobs([ server ])
-
-      const { availableJobs } = await server.runnerJobs.requestLive({ runnerToken })
-      expect(availableJobs).to.have.lengthOf(0)
-
-      await stopFfmpeg(ffmpegCommand)
-    })
-  })
-
-  describe('With transcoding enabled on classic live', function () {
-    let live: LiveVideo
-    let video: Video
-    let ffmpegCommand: FfmpegCommand
-    let jobUUID: string
-    let acceptedJob: RunnerJob & { jobToken: string }
-
-    async function testPlaylistFile (fixture: string, expected: string) {
-      const text = await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/${fixture}` })
-      expect(await readFile(buildAbsoluteFixturePath(expected), 'utf-8')).to.equal(text)
-
-    }
-
-    async function testTSFile (fixture: string, expected: string) {
-      const { body } = await makeRawRequest({ url: `${baseUrl}/${video.uuid}/${fixture}`, expectedStatus: HttpStatusCode.OK_200 })
-      expect(await readFile(buildAbsoluteFixturePath(expected))).to.deep.equal(body)
-    }
-
-    before(async function () {
-      await server.config.enableLive({
-        allowReplay: true,
-        resolutions: 'max',
-        transcoding: true
-      })
-    })
-
-    it('Should publish a a live and have available jobs', async function () {
-      this.timeout(120000)
-
-      const data = await server.live.quickCreate({ permanentLive: false, saveReplay: false, privacy: VideoPrivacy.PUBLIC })
-      live = data.live
-      video = data.video
-
-      ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
-      await waitJobs([ server ])
-
-      const job = await server.runnerJobs.requestLiveJob(runnerToken)
-      jobUUID = job.uuid
-
-      expect(job.type).to.equal('live-rtmp-hls-transcoding')
-      expect(job.payload.input.rtmpUrl).to.exist
-
-      expect(job.payload.output.toTranscode).to.have.lengthOf(5)
-
-      for (const { resolution, fps } of job.payload.output.toTranscode) {
-        expect([ 720, 480, 360, 240, 144 ]).to.contain(resolution)
-
-        expect(fps).to.be.above(25)
-        expect(fps).to.be.below(70)
-      }
-    })
-
-    it('Should update the live with a new chunk', async function () {
-      this.timeout(120000)
-
-      const { job } = await server.runnerJobs.accept<RunnerJobLiveRTMPHLSTranscodingPayload>({ jobUUID, runnerToken })
-      acceptedJob = job
-
-      {
-        const payload: LiveRTMPHLSTranscodingUpdatePayload = {
-          masterPlaylistFile: 'live/master.m3u8',
-          resolutionPlaylistFile: 'live/0.m3u8',
-          resolutionPlaylistFilename: '0.m3u8',
-          type: 'add-chunk',
-          videoChunkFile: 'live/0-000067.ts',
-          videoChunkFilename: '0-000067.ts'
-        }
-        await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: job.jobToken, payload, progress: 50 })
-
-        const updatedJob = await server.runnerJobs.getJob({ uuid: job.uuid })
-        expect(updatedJob.progress).to.equal(50)
-      }
-
-      {
-        const payload: LiveRTMPHLSTranscodingUpdatePayload = {
-          resolutionPlaylistFile: 'live/1.m3u8',
-          resolutionPlaylistFilename: '1.m3u8',
-          type: 'add-chunk',
-          videoChunkFile: 'live/1-000068.ts',
-          videoChunkFilename: '1-000068.ts'
-        }
-        await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: job.jobToken, payload })
-      }
-
-      await wait(1000)
-
-      await testPlaylistFile('master.m3u8', 'live/master.m3u8')
-      await testPlaylistFile('0.m3u8', 'live/0.m3u8')
-      await testPlaylistFile('1.m3u8', 'live/1.m3u8')
-
-      await testTSFile('0-000067.ts', 'live/0-000067.ts')
-      await testTSFile('1-000068.ts', 'live/1-000068.ts')
-    })
-
-    it('Should replace existing m3u8 on update', async function () {
-      this.timeout(120000)
-
-      const payload: LiveRTMPHLSTranscodingUpdatePayload = {
-        masterPlaylistFile: 'live/1.m3u8',
-        resolutionPlaylistFilename: '0.m3u8',
-        resolutionPlaylistFile: 'live/1.m3u8',
-        type: 'add-chunk',
-        videoChunkFile: 'live/1-000069.ts',
-        videoChunkFilename: '1-000068.ts'
-      }
-      await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload })
-      await wait(1000)
-
-      await testPlaylistFile('master.m3u8', 'live/1.m3u8')
-      await testPlaylistFile('0.m3u8', 'live/1.m3u8')
-      await testTSFile('1-000068.ts', 'live/1-000069.ts')
-    })
-
-    it('Should update the live with removed chunks', async function () {
-      this.timeout(120000)
-
-      const payload: LiveRTMPHLSTranscodingUpdatePayload = {
-        resolutionPlaylistFile: 'live/0.m3u8',
-        resolutionPlaylistFilename: '0.m3u8',
-        type: 'remove-chunk',
-        videoChunkFilename: '1-000068.ts'
-      }
-      await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload })
-
-      await wait(1000)
-
-      await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/master.m3u8` })
-      await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/0.m3u8` })
-      await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/1.m3u8` })
-      await makeRawRequest({ url: `${baseUrl}/${video.uuid}/0-000067.ts`, expectedStatus: HttpStatusCode.OK_200 })
-      await makeRawRequest({ url: `${baseUrl}/${video.uuid}/1-000068.ts`, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
-    })
-
-    it('Should complete the live and save the replay', async function () {
-      this.timeout(120000)
-
-      for (const segment of [ '0-000069.ts', '0-000070.ts' ]) {
-        const payload: LiveRTMPHLSTranscodingUpdatePayload = {
-          masterPlaylistFile: 'live/master.m3u8',
-          resolutionPlaylistFilename: '0.m3u8',
-          resolutionPlaylistFile: 'live/0.m3u8',
-          type: 'add-chunk',
-          videoChunkFile: 'live/' + segment,
-          videoChunkFilename: segment
-        }
-        await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload })
-
-        await wait(1000)
-      }
-
-      await waitJobs([ server ])
-
-      {
-        const { state } = await server.videos.get({ id: video.uuid })
-        expect(state.id).to.equal(VideoState.PUBLISHED)
-      }
-
-      await stopFfmpeg(ffmpegCommand)
-
-      await server.runnerJobs.success({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload: {} })
-
-      await wait(1500)
-      await waitJobs([ server ])
-
-      {
-        const { state } = await server.videos.get({ id: video.uuid })
-        expect(state.id).to.equal(VideoState.LIVE_ENDED)
-
-        const session = await server.live.findLatestSession({ videoId: video.uuid })
-        expect(session.error).to.be.null
-      }
-    })
-  })
-
-  describe('With transcoding enabled on cancelled/aborted/errored live', function () {
-    let live: LiveVideo
-    let video: Video
-    let ffmpegCommand: FfmpegCommand
-
-    async function prepare () {
-      ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
-      await server.runnerJobs.requestLiveJob(runnerToken)
-
-      const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'live-rtmp-hls-transcoding' })
-
-      return job
-    }
-
-    async function checkSessionError (error: LiveVideoError) {
-      await wait(1500)
-      await waitJobs([ server ])
-
-      const session = await server.live.findLatestSession({ videoId: video.uuid })
-      expect(session.error).to.equal(error)
-    }
-
-    before(async function () {
-      await server.config.enableLive({
-        allowReplay: true,
-        resolutions: 'max',
-        transcoding: true
-      })
-
-      const data = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC })
-      live = data.live
-      video = data.video
-    })
-
-    it('Should abort a running live', async function () {
-      this.timeout(120000)
-
-      const job = await prepare()
-
-      await Promise.all([
-        server.runnerJobs.abort({ jobUUID: job.uuid, runnerToken, jobToken: job.jobToken, reason: 'abort' }),
-        testFfmpegStreamError(ffmpegCommand, true)
-      ])
-
-      // Abort is not supported
-      await checkSessionError(LiveVideoError.RUNNER_JOB_ERROR)
-    })
-
-    it('Should cancel a running live', async function () {
-      this.timeout(120000)
-
-      const job = await prepare()
-
-      await Promise.all([
-        server.runnerJobs.cancelByAdmin({ jobUUID: job.uuid }),
-        testFfmpegStreamError(ffmpegCommand, true)
-      ])
-
-      await checkSessionError(LiveVideoError.RUNNER_JOB_CANCEL)
-    })
-
-    it('Should error a running live', async function () {
-      this.timeout(120000)
-
-      const job = await prepare()
-
-      await Promise.all([
-        server.runnerJobs.error({ jobUUID: job.uuid, runnerToken, jobToken: job.jobToken, message: 'error' }),
-        testFfmpegStreamError(ffmpegCommand, true)
-      ])
-
-      await checkSessionError(LiveVideoError.RUNNER_JOB_ERROR)
-    })
-  })
-
-  after(async function () {
-    await cleanupTests([ server ])
-  })
-})