Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

1 files changed, 0 insertions, 159 deletions

diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
deleted file mode 100644
index 9a99b6722..000000000
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ /dev/null
@@ -1,159 +0,0 @@
-import { Job } from 'bullmq'
-import { remove } from 'fs-extra'
-import { join } from 'path'
-import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { updateTorrentMetadata } from '@server/helpers/webtorrent'
-import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
-import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage'
-import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
-import { VideoPathManager } from '@server/lib/video-path-manager'
-import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
-import { VideoModel } from '@server/models/video/video'
-import { VideoJobInfoModel } from '@server/models/video/video-job-info'
-import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
-import { MoveObjectStoragePayload, VideoState, VideoStorage } from '@shared/models'
-
-const lTagsBase = loggerTagsFactory('move-object-storage')
-
-export async function processMoveToObjectStorage (job: Job) {
-  const payload = job.data as MoveObjectStoragePayload
-  logger.info('Moving video %s in job %s.', payload.videoUUID, job.id)
-
-  const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
-
-  const video = await VideoModel.loadWithFiles(payload.videoUUID)
-  // No video, maybe deleted?
-  if (!video) {
-    logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
-    fileMutexReleaser()
-    return undefined
-  }
-
-  const lTags = lTagsBase(video.uuid, video.url)
-
-  try {
-    if (video.VideoFiles) {
-      logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
-
-      await moveWebVideoFiles(video)
-    }
-
-    if (video.VideoStreamingPlaylists) {
-      logger.debug('Moving HLS playlist of %s.', video.uuid)
-
-      await moveHLSFiles(video)
-    }
-
-    const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
-    if (pendingMove === 0) {
-      logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags)
-
-      await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
-    }
-  } catch (err) {
-    await onMoveToObjectStorageFailure(job, err)
-
-    throw err
-  } finally {
-    fileMutexReleaser()
-  }
-
-  return payload.videoUUID
-}
-
-export async function onMoveToObjectStorageFailure (job: Job, err: any) {
-  const payload = job.data as MoveObjectStoragePayload
-
-  const video = await VideoModel.loadWithFiles(payload.videoUUID)
-  if (!video) return
-
-  logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) })
-
-  await moveToFailedMoveToObjectStorageState(video)
-  await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
-}
-
-// ---------------------------------------------------------------------------
-
-async function moveWebVideoFiles (video: MVideoWithAllFiles) {
-  for (const file of video.VideoFiles) {
-    if (file.storage !== VideoStorage.FILE_SYSTEM) continue
-
-    const fileUrl = await storeWebVideoFile(video, file)
-
-    const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
-    await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
-  }
-}
-
-async function moveHLSFiles (video: MVideoWithAllFiles) {
-  for (const playlist of video.VideoStreamingPlaylists) {
-    const playlistWithVideo = playlist.withVideo(video)
-
-    for (const file of playlist.VideoFiles) {
-      if (file.storage !== VideoStorage.FILE_SYSTEM) continue
-
-      // Resolution playlist
-      const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
-      await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
-
-      // Resolution fragmented file
-      const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
-
-      const oldPath = join(getHLSDirectory(video), file.filename)
-
-      await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
-    }
-  }
-}
-
-async function doAfterLastJob (options: {
-  video: MVideoWithAllFiles
-  previousVideoState: VideoState
-  isNewVideo: boolean
-}) {
-  const { video, previousVideoState, isNewVideo } = options
-
-  for (const playlist of video.VideoStreamingPlaylists) {
-    if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
-
-    const playlistWithVideo = playlist.withVideo(video)
-
-    // Master playlist
-    playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
-    // Sha256 segments file
-    playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
-
-    playlist.storage = VideoStorage.OBJECT_STORAGE
-
-    playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
-    playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
-
-    await playlist.save()
-  }
-
-  // Remove empty hls video directory
-  if (video.VideoStreamingPlaylists) {
-    await remove(getHLSDirectory(video))
-  }
-
-  await moveToNextState({ video, previousVideoState, isNewVideo })
-}
-
-async function onFileMoved (options: {
-  videoOrPlaylist: MVideo | MStreamingPlaylistVideo
-  file: MVideoFile
-  fileUrl: string
-  oldPath: string
-}) {
-  const { videoOrPlaylist, file, fileUrl, oldPath } = options
-
-  file.fileUrl = fileUrl
-  file.storage = VideoStorage.OBJECT_STORAGE
-
-  await updateTorrentMetadata(videoOrPlaylist, file)
-  await file.save()
-
-  logger.debug('Removing %s because it\'s now on object storage', oldPath)
-  await remove(oldPath)
-}