Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

5 files changed, 0 insertions, 611 deletions

diff --git a/server/models/video/formatter/index.ts b/server/models/video/formatter/index.ts
deleted file mode 100644
index 77b406559..000000000
--- a/server/models/video/formatter/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export * from './video-activity-pub-format'
-export * from './video-api-format'
diff --git a/server/models/video/formatter/shared/index.ts b/server/models/video/formatter/shared/index.ts
deleted file mode 100644
index d558fa7d6..000000000
--- a/server/models/video/formatter/shared/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from './video-format-utils'
diff --git a/server/models/video/formatter/shared/video-format-utils.ts b/server/models/video/formatter/shared/video-format-utils.ts
deleted file mode 100644
index df3bbdf1c..000000000
--- a/server/models/video/formatter/shared/video-format-utils.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { MVideoFile } from '@server/types/models'
-
-export function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) {
-  if (fileA.resolution < fileB.resolution) return 1
-  if (fileA.resolution === fileB.resolution) return 0
-  return -1
-}
diff --git a/server/models/video/formatter/video-activity-pub-format.ts b/server/models/video/formatter/video-activity-pub-format.ts
deleted file mode 100644
index 694c66c33..000000000
--- a/server/models/video/formatter/video-activity-pub-format.ts
+++ /dev/null
@@ -1,296 +0,0 @@
-import { isArray } from '@server/helpers/custom-validators/misc'
-import { generateMagnetUri } from '@server/helpers/webtorrent'
-import { getActivityStreamDuration } from '@server/lib/activitypub/activity'
-import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
-import {
-  ActivityIconObject,
-  ActivityPlaylistUrlObject,
-  ActivityPubStoryboard,
-  ActivityTagObject,
-  ActivityTrackerUrlObject,
-  ActivityUrlObject,
-  VideoObject
-} from '@shared/models'
-import { MIMETYPES, WEBSERVER } from '../../../initializers/constants'
-import {
-  getLocalVideoCommentsActivityPubUrl,
-  getLocalVideoDislikesActivityPubUrl,
-  getLocalVideoLikesActivityPubUrl,
-  getLocalVideoSharesActivityPubUrl
-} from '../../../lib/activitypub/url'
-import { MStreamingPlaylistFiles, MUserId, MVideo, MVideoAP, MVideoFile } from '../../../types/models'
-import { VideoCaptionModel } from '../video-caption'
-import { sortByResolutionDesc } from './shared'
-import { getCategoryLabel, getLanguageLabel, getLicenceLabel } from './video-api-format'
-
-export function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
-  const language = video.language
-    ? { identifier: video.language, name: getLanguageLabel(video.language) }
-    : undefined
-
-  const category = video.category
-    ? { identifier: video.category + '', name: getCategoryLabel(video.category) }
-    : undefined
-
-  const licence = video.licence
-    ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) }
-    : undefined
-
-  const url: ActivityUrlObject[] = [
-    // HTML url should be the first element in the array so Mastodon correctly displays the embed
-    {
-      type: 'Link',
-      mediaType: 'text/html',
-      href: WEBSERVER.URL + '/videos/watch/' + video.uuid
-    } as ActivityUrlObject,
-
-    ...buildVideoFileUrls({ video, files: video.VideoFiles }),
-
-    ...buildStreamingPlaylistUrls(video),
-
-    ...buildTrackerUrls(video)
-  ]
-
-  return {
-    type: 'Video' as 'Video',
-    id: video.url,
-    name: video.name,
-    duration: getActivityStreamDuration(video.duration),
-    uuid: video.uuid,
-    category,
-    licence,
-    language,
-    views: video.views,
-    sensitive: video.nsfw,
-    waitTranscoding: video.waitTranscoding,
-
-    state: video.state,
-    commentsEnabled: video.commentsEnabled,
-    downloadEnabled: video.downloadEnabled,
-    published: video.publishedAt.toISOString(),
-
-    originallyPublishedAt: video.originallyPublishedAt
-      ? video.originallyPublishedAt.toISOString()
-      : null,
-
-    updated: video.updatedAt.toISOString(),
-
-    uploadDate: video.inputFileUpdatedAt?.toISOString(),
-
-    tag: buildTags(video),
-
-    mediaType: 'text/markdown',
-    content: video.description,
-    support: video.support,
-
-    subtitleLanguage: buildSubtitleLanguage(video),
-
-    icon: buildIcon(video),
-
-    preview: buildPreviewAPAttribute(video),
-
-    url,
-
-    likes: getLocalVideoLikesActivityPubUrl(video),
-    dislikes: getLocalVideoDislikesActivityPubUrl(video),
-    shares: getLocalVideoSharesActivityPubUrl(video),
-    comments: getLocalVideoCommentsActivityPubUrl(video),
-
-    attributedTo: [
-      {
-        type: 'Person',
-        id: video.VideoChannel.Account.Actor.url
-      },
-      {
-        type: 'Group',
-        id: video.VideoChannel.Actor.url
-      }
-    ],
-
-    ...buildLiveAPAttributes(video)
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Private
-// ---------------------------------------------------------------------------
-
-function buildLiveAPAttributes (video: MVideoAP) {
-  if (!video.isLive) {
-    return {
-      isLiveBroadcast: false,
-      liveSaveReplay: null,
-      permanentLive: null,
-      latencyMode: null
-    }
-  }
-
-  return {
-    isLiveBroadcast: true,
-    liveSaveReplay: video.VideoLive.saveReplay,
-    permanentLive: video.VideoLive.permanentLive,
-    latencyMode: video.VideoLive.latencyMode
-  }
-}
-
-function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {
-  if (!video.Storyboard) return undefined
-
-  const storyboard = video.Storyboard
-
-  return [
-    {
-      type: 'Image',
-      rel: [ 'storyboard' ],
-      url: [
-        {
-          mediaType: 'image/jpeg',
-
-          href: storyboard.getOriginFileUrl(video),
-
-          width: storyboard.totalWidth,
-          height: storyboard.totalHeight,
-
-          tileWidth: storyboard.spriteWidth,
-          tileHeight: storyboard.spriteHeight,
-          tileDuration: getActivityStreamDuration(storyboard.spriteDuration)
-        }
-      ]
-    }
-  ]
-}
-
-function buildVideoFileUrls (options: {
-  video: MVideo
-  files: MVideoFile[]
-  user?: MUserId
-}): ActivityUrlObject[] {
-  const { video, files } = options
-
-  if (!isArray(files)) return []
-
-  const urls: ActivityUrlObject[] = []
-
-  const trackerUrls = video.getTrackerUrls()
-  const sortedFiles = files
-    .filter(f => !f.isLive())
-    .sort(sortByResolutionDesc)
-
-  for (const file of sortedFiles) {
-    urls.push({
-      type: 'Link',
-      mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] as any,
-      href: file.getFileUrl(video),
-      height: file.resolution,
-      size: file.size,
-      fps: file.fps
-    })
-
-    urls.push({
-      type: 'Link',
-      rel: [ 'metadata', MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] ],
-      mediaType: 'application/json' as 'application/json',
-      href: getLocalVideoFileMetadataUrl(video, file),
-      height: file.resolution,
-      fps: file.fps
-    })
-
-    if (file.hasTorrent()) {
-      urls.push({
-        type: 'Link',
-        mediaType: 'application/x-bittorrent' as 'application/x-bittorrent',
-        href: file.getTorrentUrl(),
-        height: file.resolution
-      })
-
-      urls.push({
-        type: 'Link',
-        mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
-        href: generateMagnetUri(video, file, trackerUrls),
-        height: file.resolution
-      })
-    }
-  }
-
-  return urls
-}
-
-// ---------------------------------------------------------------------------
-
-function buildStreamingPlaylistUrls (video: MVideoAP): ActivityPlaylistUrlObject[] {
-  if (!isArray(video.VideoStreamingPlaylists)) return []
-
-  return video.VideoStreamingPlaylists
-    .map(playlist => ({
-      type: 'Link',
-      mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
-      href: playlist.getMasterPlaylistUrl(video),
-      tag: buildStreamingPlaylistTags(video, playlist)
-    }))
-}
-
-function buildStreamingPlaylistTags (video: MVideoAP, playlist: MStreamingPlaylistFiles) {
-  return [
-    ...playlist.p2pMediaLoaderInfohashes.map(i => ({ type: 'Infohash' as 'Infohash', name: i })),
-
-    {
-      type: 'Link',
-      name: 'sha256',
-      mediaType: 'application/json' as 'application/json',
-      href: playlist.getSha256SegmentsUrl(video)
-    },
-
-    ...buildVideoFileUrls({ video, files: playlist.VideoFiles })
-  ] as ActivityTagObject[]
-}
-
-// ---------------------------------------------------------------------------
-
-function buildTrackerUrls (video: MVideoAP): ActivityTrackerUrlObject[] {
-  return video.getTrackerUrls()
-    .map(trackerUrl => {
-      const rel2 = trackerUrl.startsWith('http')
-        ? 'http'
-        : 'websocket'
-
-      return {
-        type: 'Link',
-        name: `tracker-${rel2}`,
-        rel: [ 'tracker', rel2 ],
-        href: trackerUrl
-      }
-    })
-}
-
-// ---------------------------------------------------------------------------
-
-function buildTags (video: MVideoAP) {
-  if (!isArray(video.Tags)) return []
-
-  return video.Tags.map(t => ({
-    type: 'Hashtag' as 'Hashtag',
-    name: t.name
-  }))
-}
-
-function buildIcon (video: MVideoAP): ActivityIconObject[] {
-  return [ video.getMiniature(), video.getPreview() ]
-    .map(i => ({
-      type: 'Image',
-      url: i.getOriginFileUrl(video),
-      mediaType: 'image/jpeg',
-      width: i.width,
-      height: i.height
-    }))
-}
-
-function buildSubtitleLanguage (video: MVideoAP) {
-  if (!isArray(video.VideoCaptions)) return []
-
-  return video.VideoCaptions
-    .map(caption => ({
-      identifier: caption.language,
-      name: VideoCaptionModel.getLanguageLabel(caption.language),
-      url: caption.getFileUrl(video)
-    }))
-}
diff --git a/server/models/video/formatter/video-api-format.ts b/server/models/video/formatter/video-api-format.ts
deleted file mode 100644
index 7a58f5d3c..000000000
--- a/server/models/video/formatter/video-api-format.ts
+++ /dev/null
@@ -1,305 +0,0 @@
-import { generateMagnetUri } from '@server/helpers/webtorrent'
-import { tracer } from '@server/lib/opentelemetry/tracing'
-import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
-import { VideoViewsManager } from '@server/lib/views/video-views-manager'
-import { uuidToShort } from '@shared/extra-utils'
-import {
-  Video,
-  VideoAdditionalAttributes,
-  VideoDetails,
-  VideoFile,
-  VideoInclude,
-  VideosCommonQueryAfterSanitize,
-  VideoStreamingPlaylist
-} from '@shared/models'
-import { isArray } from '../../../helpers/custom-validators/misc'
-import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES, VIDEO_STATES } from '../../../initializers/constants'
-import { MServer, MStreamingPlaylistRedundanciesOpt, MVideoFormattable, MVideoFormattableDetails } from '../../../types/models'
-import { MVideoFileRedundanciesOpt } from '../../../types/models/video/video-file'
-import { sortByResolutionDesc } from './shared'
-
-export type VideoFormattingJSONOptions = {
-  completeDescription?: boolean
-
-  additionalAttributes?: {
-    state?: boolean
-    waitTranscoding?: boolean
-    scheduledUpdate?: boolean
-    blacklistInfo?: boolean
-    files?: boolean
-    blockedOwner?: boolean
-  }
-}
-
-export function guessAdditionalAttributesFromQuery (query: VideosCommonQueryAfterSanitize): VideoFormattingJSONOptions {
-  if (!query?.include) return {}
-
-  return {
-    additionalAttributes: {
-      state: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE),
-      waitTranscoding: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE),
-      scheduledUpdate: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE),
-      blacklistInfo: !!(query.include & VideoInclude.BLACKLISTED),
-      files: !!(query.include & VideoInclude.FILES),
-      blockedOwner: !!(query.include & VideoInclude.BLOCKED_OWNER)
-    }
-  }
-}
-
-// ---------------------------------------------------------------------------
-
-export function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoFormattingJSONOptions = {}): Video {
-  const span = tracer.startSpan('peertube.VideoModel.toFormattedJSON')
-
-  const userHistory = isArray(video.UserVideoHistories)
-    ? video.UserVideoHistories[0]
-    : undefined
-
-  const videoObject: Video = {
-    id: video.id,
-    uuid: video.uuid,
-    shortUUID: uuidToShort(video.uuid),
-
-    url: video.url,
-
-    name: video.name,
-    category: {
-      id: video.category,
-      label: getCategoryLabel(video.category)
-    },
-    licence: {
-      id: video.licence,
-      label: getLicenceLabel(video.licence)
-    },
-    language: {
-      id: video.language,
-      label: getLanguageLabel(video.language)
-    },
-    privacy: {
-      id: video.privacy,
-      label: getPrivacyLabel(video.privacy)
-    },
-    nsfw: video.nsfw,
-
-    truncatedDescription: video.getTruncatedDescription(),
-    description: options && options.completeDescription === true
-      ? video.description
-      : video.getTruncatedDescription(),
-
-    isLocal: video.isOwned(),
-    duration: video.duration,
-
-    views: video.views,
-    viewers: VideoViewsManager.Instance.getViewers(video),
-
-    likes: video.likes,
-    dislikes: video.dislikes,
-    thumbnailPath: video.getMiniatureStaticPath(),
-    previewPath: video.getPreviewStaticPath(),
-    embedPath: video.getEmbedStaticPath(),
-    createdAt: video.createdAt,
-    updatedAt: video.updatedAt,
-    publishedAt: video.publishedAt,
-    originallyPublishedAt: video.originallyPublishedAt,
-
-    isLive: video.isLive,
-
-    account: video.VideoChannel.Account.toFormattedSummaryJSON(),
-    channel: video.VideoChannel.toFormattedSummaryJSON(),
-
-    userHistory: userHistory
-      ? { currentTime: userHistory.currentTime }
-      : undefined,
-
-    // Can be added by external plugins
-    pluginData: (video as any).pluginData,
-
-    ...buildAdditionalAttributes(video, options)
-  }
-
-  span.end()
-
-  return videoObject
-}
-
-export function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): VideoDetails {
-  const span = tracer.startSpan('peertube.VideoModel.toFormattedDetailsJSON')
-
-  const videoJSON = video.toFormattedJSON({
-    completeDescription: true,
-    additionalAttributes: {
-      scheduledUpdate: true,
-      blacklistInfo: true,
-      files: true
-    }
-  }) as Video & Required<Pick<Video, 'files' | 'streamingPlaylists' | 'scheduledUpdate' | 'blacklisted' | 'blacklistedReason'>>
-
-  const tags = video.Tags
-    ? video.Tags.map(t => t.name)
-    : []
-
-  const detailsJSON = {
-    ...videoJSON,
-
-    support: video.support,
-    descriptionPath: video.getDescriptionAPIPath(),
-    channel: video.VideoChannel.toFormattedJSON(),
-    account: video.VideoChannel.Account.toFormattedJSON(),
-    tags,
-    commentsEnabled: video.commentsEnabled,
-    downloadEnabled: video.downloadEnabled,
-    waitTranscoding: video.waitTranscoding,
-    inputFileUpdatedAt: video.inputFileUpdatedAt,
-    state: {
-      id: video.state,
-      label: getStateLabel(video.state)
-    },
-
-    trackerUrls: video.getTrackerUrls()
-  }
-
-  span.end()
-
-  return detailsJSON
-}
-
-export function streamingPlaylistsModelToFormattedJSON (
-  video: MVideoFormattable,
-  playlists: MStreamingPlaylistRedundanciesOpt[]
-): VideoStreamingPlaylist[] {
-  if (isArray(playlists) === false) return []
-
-  return playlists
-    .map(playlist => ({
-      id: playlist.id,
-      type: playlist.type,
-
-      playlistUrl: playlist.getMasterPlaylistUrl(video),
-      segmentsSha256Url: playlist.getSha256SegmentsUrl(video),
-
-      redundancies: isArray(playlist.RedundancyVideos)
-        ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl }))
-        : [],
-
-      files: videoFilesModelToFormattedJSON(video, playlist.VideoFiles)
-    }))
-}
-
-export function videoFilesModelToFormattedJSON (
-  video: MVideoFormattable,
-  videoFiles: MVideoFileRedundanciesOpt[],
-  options: {
-    includeMagnet?: boolean // default true
-  } = {}
-): VideoFile[] {
-  const { includeMagnet = true } = options
-
-  if (isArray(videoFiles) === false) return []
-
-  const trackerUrls = includeMagnet
-    ? video.getTrackerUrls()
-    : []
-
-  return videoFiles
-    .filter(f => !f.isLive())
-    .sort(sortByResolutionDesc)
-    .map(videoFile => {
-      return {
-        id: videoFile.id,
-
-        resolution: {
-          id: videoFile.resolution,
-          label: videoFile.resolution === 0
-            ? 'Audio'
-            : `${videoFile.resolution}p`
-        },
-
-        magnetUri: includeMagnet && videoFile.hasTorrent()
-          ? generateMagnetUri(video, videoFile, trackerUrls)
-          : undefined,
-
-        size: videoFile.size,
-        fps: videoFile.fps,
-
-        torrentUrl: videoFile.getTorrentUrl(),
-        torrentDownloadUrl: videoFile.getTorrentDownloadUrl(),
-
-        fileUrl: videoFile.getFileUrl(video),
-        fileDownloadUrl: videoFile.getFileDownloadUrl(video),
-
-        metadataUrl: videoFile.metadataUrl ?? getLocalVideoFileMetadataUrl(video, videoFile)
-      }
-    })
-}
-
-// ---------------------------------------------------------------------------
-
-export function getCategoryLabel (id: number) {
-  return VIDEO_CATEGORIES[id] || 'Unknown'
-}
-
-export function getLicenceLabel (id: number) {
-  return VIDEO_LICENCES[id] || 'Unknown'
-}
-
-export function getLanguageLabel (id: string) {
-  return VIDEO_LANGUAGES[id] || 'Unknown'
-}
-
-export function getPrivacyLabel (id: number) {
-  return VIDEO_PRIVACIES[id] || 'Unknown'
-}
-
-export function getStateLabel (id: number) {
-  return VIDEO_STATES[id] || 'Unknown'
-}
-
-// ---------------------------------------------------------------------------
-// Private
-// ---------------------------------------------------------------------------
-
-function buildAdditionalAttributes (video: MVideoFormattable, options: VideoFormattingJSONOptions) {
-  const add = options.additionalAttributes
-
-  const result: Partial<VideoAdditionalAttributes> = {}
-
-  if (add?.state === true) {
-    result.state = {
-      id: video.state,
-      label: getStateLabel(video.state)
-    }
-  }
-
-  if (add?.waitTranscoding === true) {
-    result.waitTranscoding = video.waitTranscoding
-  }
-
-  if (add?.scheduledUpdate === true && video.ScheduleVideoUpdate) {
-    result.scheduledUpdate = {
-      updateAt: video.ScheduleVideoUpdate.updateAt,
-      privacy: video.ScheduleVideoUpdate.privacy || undefined
-    }
-  }
-
-  if (add?.blacklistInfo === true) {
-    result.blacklisted = !!video.VideoBlacklist
-    result.blacklistedReason =
-      video.VideoBlacklist
-        ? video.VideoBlacklist.reason
-        : null
-  }
-
-  if (add?.blockedOwner === true) {
-    result.blockedOwner = video.VideoChannel.Account.isBlocked()
-
-    const server = video.VideoChannel.Account.Actor.Server as MServer
-    result.blockedServer = !!(server?.isBlocked())
-  }
-
-  if (add?.files === true) {
-    result.streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists)
-    result.files = videoFilesModelToFormattedJSON(video, video.VideoFiles)
-  }
-
-  return result
-}