Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

3 files changed, 0 insertions, 299 deletions

diff --git a/scripts/migrations/peertube-4.0.ts b/scripts/migrations/peertube-4.0.ts
deleted file mode 100644
index b0891c2e6..000000000
--- a/scripts/migrations/peertube-4.0.ts
+++ /dev/null
@@ -1,104 +0,0 @@
-import Bluebird from 'bluebird'
-import { move, readFile, writeFile } from 'fs-extra'
-import { join } from 'path'
-import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
-import { JobQueue } from '@server/lib/job-queue'
-import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
-import { VideoPathManager } from '@server/lib/video-path-manager'
-import { VideoModel } from '@server/models/video/video'
-import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
-import { initDatabaseModels } from '../../server/initializers/database'
-
-run()
-  .then(() => process.exit(0))
-  .catch(err => {
-    console.error(err)
-    process.exit(-1)
-  })
-
-async function run () {
-  console.log('Migrate old HLS paths to new format.')
-
-  await initDatabaseModels(true)
-
-  JobQueue.Instance.init()
-
-  const ids = await VideoModel.listLocalIds()
-
-  await Bluebird.map(ids, async id => {
-    try {
-      await processVideo(id)
-    } catch (err) {
-      console.error('Cannot process video %s.', { err })
-    }
-  }, { concurrency: 5 })
-
-  console.log('Migration finished!')
-}
-
-async function processVideo (videoId: number) {
-  const video = await VideoModel.loadWithFiles(videoId)
-
-  const hls = video.getHLSPlaylist()
-  if (video.isLive || !hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) {
-    return
-  }
-
-  console.log(`Renaming HLS playlist files of video ${video.name}.`)
-
-  const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
-  const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video)
-
-  const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename)
-  let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8')
-
-  for (const videoFile of hls.VideoFiles) {
-    const srcName = `${videoFile.resolution}.m3u8`
-    const dstName = getHlsResolutionPlaylistFilename(videoFile.filename)
-
-    const src = join(hlsDirPath, srcName)
-    const dst = join(hlsDirPath, dstName)
-
-    try {
-      await move(src, dst)
-
-      masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName)
-    } catch (err) {
-      console.error('Cannot move video file %s to %s.', src, dst, err)
-    }
-  }
-
-  await writeFile(masterPlaylistPath, masterPlaylistContent)
-
-  if (playlist.segmentsSha256Filename === 'segments-sha256.json') {
-    try {
-      const newName = generateHlsSha256SegmentsFilename(video.isLive)
-
-      const dst = join(hlsDirPath, newName)
-      await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst)
-      playlist.segmentsSha256Filename = newName
-    } catch (err) {
-      console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err)
-    }
-  }
-
-  if (playlist.playlistFilename === 'master.m3u8') {
-    try {
-      const newName = generateHLSMasterPlaylistFilename(video.isLive)
-
-      const dst = join(hlsDirPath, newName)
-      await move(join(hlsDirPath, playlist.playlistFilename), dst)
-      playlist.playlistFilename = newName
-    } catch (err) {
-      console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err)
-    }
-  }
-
-  // Everything worked, we can save the playlist now
-  await playlist.save()
-
-  const allVideo = await VideoModel.loadFull(video.id)
-  await federateVideoIfNeeded(allVideo, false)
-
-  console.log(`Successfully moved HLS files of ${video.name}.`)
-}
diff --git a/scripts/migrations/peertube-4.2.ts b/scripts/migrations/peertube-4.2.ts
deleted file mode 100644
index d8929692b..000000000
--- a/scripts/migrations/peertube-4.2.ts
+++ /dev/null
@@ -1,124 +0,0 @@
-import { minBy } from 'lodash'
-import { join } from 'path'
-import { getImageSize, processImage } from '@server/helpers/image-utils'
-import { CONFIG } from '@server/initializers/config'
-import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants'
-import { updateActorImages } from '@server/lib/activitypub/actors'
-import { sendUpdateActor } from '@server/lib/activitypub/send'
-import { getBiggestActorImage } from '@server/lib/actor-image'
-import { JobQueue } from '@server/lib/job-queue'
-import { AccountModel } from '@server/models/account/account'
-import { ActorModel } from '@server/models/actor/actor'
-import { VideoChannelModel } from '@server/models/video/video-channel'
-import { MAccountDefault, MActorDefault, MChannelDefault } from '@server/types/models'
-import { getLowercaseExtension } from '@shared/core-utils'
-import { buildUUID } from '@shared/extra-utils'
-import { ActorImageType } from '@shared/models'
-import { initDatabaseModels } from '../../server/initializers/database'
-
-run()
-  .then(() => process.exit(0))
-  .catch(err => {
-    console.error(err)
-    process.exit(-1)
-  })
-
-async function run () {
-  console.log('Generate avatar miniatures from existing avatars.')
-
-  await initDatabaseModels(true)
-  JobQueue.Instance.init()
-
-  const accounts: AccountModel[] = await AccountModel.findAll({
-    include: [
-      {
-        model: ActorModel,
-        required: true,
-        where: {
-          serverId: null
-        }
-      },
-      {
-        model: VideoChannelModel,
-        include: [
-          {
-            model: AccountModel
-          }
-        ]
-      }
-    ]
-  })
-
-  for (const account of accounts) {
-    try {
-      await fillAvatarSizeIfNeeded(account)
-      await generateSmallerAvatarIfNeeded(account)
-    } catch (err) {
-      console.error(`Cannot process account avatar ${account.name}`, err)
-    }
-
-    for (const videoChannel of account.VideoChannels) {
-      try {
-        await fillAvatarSizeIfNeeded(videoChannel)
-        await generateSmallerAvatarIfNeeded(videoChannel)
-      } catch (err) {
-        console.error(`Cannot process channel avatar ${videoChannel.name}`, err)
-      }
-    }
-  }
-
-  console.log('Generation finished!')
-}
-
-async function fillAvatarSizeIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) {
-  const avatars = accountOrChannel.Actor.Avatars
-
-  for (const avatar of avatars) {
-    if (avatar.width && avatar.height) continue
-
-    console.log('Filling size of avatars of %s.', accountOrChannel.name)
-
-    const { width, height } = await getImageSize(join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, avatar.filename))
-    avatar.width = width
-    avatar.height = height
-
-    await avatar.save()
-  }
-}
-
-async function generateSmallerAvatarIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) {
-  const avatars = accountOrChannel.Actor.Avatars
-  if (avatars.length !== 1) {
-    return
-  }
-
-  console.log(`Processing ${accountOrChannel.name}.`)
-
-  await generateSmallerAvatar(accountOrChannel.Actor)
-  accountOrChannel.Actor = Object.assign(accountOrChannel.Actor, { Server: null })
-
-  return sendUpdateActor(accountOrChannel, undefined)
-}
-
-async function generateSmallerAvatar (actor: MActorDefault) {
-  const bigAvatar = getBiggestActorImage(actor.Avatars)
-
-  const imageSize = minBy(ACTOR_IMAGES_SIZE[ActorImageType.AVATAR], 'width')
-  const sourceFilename = bigAvatar.filename
-
-  const newImageName = buildUUID() + getLowercaseExtension(sourceFilename)
-  const source = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, sourceFilename)
-  const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, newImageName)
-
-  await processImage({ path: source, destination, newSize: imageSize, keepOriginal: true })
-
-  const actorImageInfo = {
-    name: newImageName,
-    fileUrl: null,
-    height: imageSize.height,
-    width: imageSize.width,
-    onDisk: true
-  }
-
-  await updateActorImages(actor, ActorImageType.AVATAR, [ actorImageInfo ], undefined)
-}
diff --git a/scripts/migrations/peertube-5.0.ts b/scripts/migrations/peertube-5.0.ts
deleted file mode 100644
index a0f51a64c..000000000
--- a/scripts/migrations/peertube-5.0.ts
+++ /dev/null
@@ -1,71 +0,0 @@
-import { ensureDir } from 'fs-extra'
-import { Op } from 'sequelize'
-import { updateTorrentMetadata } from '@server/helpers/webtorrent'
-import { DIRECTORIES } from '@server/initializers/constants'
-import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy'
-import { VideoModel } from '@server/models/video/video'
-import { MVideoFullLight } from '@server/types/models'
-import { VideoPrivacy } from '@shared/models'
-import { initDatabaseModels } from '../../server/initializers/database'
-
-run()
-  .then(() => process.exit(0))
-  .catch(err => {
-    console.error(err)
-    process.exit(-1)
-  })
-
-async function run () {
-  console.log('Moving private video files in dedicated folders.')
-
-  await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)
-  await ensureDir(DIRECTORIES.VIDEOS.PRIVATE)
-
-  await initDatabaseModels(true)
-
-  const videos = await VideoModel.unscoped().findAll({
-    attributes: [ 'uuid' ],
-    where: {
-      privacy: {
-        [Op.in]: [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]
-      }
-    }
-  })
-
-  for (const { uuid } of videos) {
-    try {
-      console.log('Moving files of video %s.', uuid)
-
-      const video = await VideoModel.loadFull(uuid)
-
-      try {
-        await moveFilesIfPrivacyChanged(video, VideoPrivacy.PUBLIC)
-      } catch (err) {
-        console.error('Cannot move files of video %s.', uuid, err)
-      }
-
-      try {
-        await updateTorrents(video)
-      } catch (err) {
-        console.error('Cannot regenerate torrents of video %s.', uuid, err)
-      }
-    } catch (err) {
-      console.error('Cannot process video %s.', uuid, err)
-    }
-  }
-}
-
-async function updateTorrents (video: MVideoFullLight) {
-  for (const file of video.VideoFiles) {
-    await updateTorrentMetadata(video, file)
-
-    await file.save()
-  }
-
-  const playlist = video.getHLSPlaylist()
-  for (const file of (playlist?.VideoFiles || [])) {
-    await updateTorrentMetadata(playlist, file)
-
-    await file.save()
-  }
-}