Migrate server to ESM

Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:

 * Server can be faster at startup because imports() are async and we can
   easily lazy import big modules
 * Angular doesn't seem to support ES import (with .js extension), so we
   had to correctly organize peertube into a monorepo:
    * Use yarn workspace feature
    * Use typescript reference projects for dependencies
    * Shared projects have been moved into "packages", each one is now a
      node module (with a dedicated package.json/tsconfig.json)
    * server/tools have been moved into apps/ and is now a dedicated app
      bundled and published on NPM so users don't have to build peertube
      cli tools manually
    * server/tests have been moved into packages/ so we don't compile
      them every time we want to run the server
 * Use isolatedModule option:
   * Had to move from const enum to const
     (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
   * Had to explictely specify "type" imports when used in decorators
 * Prefer tsx (that uses esbuild under the hood) instead of ts-node to
   load typescript files (tests with mocha or scripts):
     * To reduce test complexity as esbuild doesn't support decorator
       metadata, we only test server files that do not import server
       models
     * We still build tests files into js files for a faster CI
 * Remove unmaintained peertube CLI import script
 * Removed some barrels to speed up execution (less imports)

5 files changed, 0 insertions, 567 deletions

diff --git a/server/lib/views/shared/index.ts b/server/lib/views/shared/index.ts
deleted file mode 100644
index 139471183..000000000
--- a/server/lib/views/shared/index.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-export * from './video-viewer-counters'
-export * from './video-viewer-stats'
-export * from './video-views'
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts
deleted file mode 100644
index f5b83130e..000000000
--- a/server/lib/views/shared/video-viewer-counters.ts
+++ /dev/null
@@ -1,198 +0,0 @@
-import { isTestOrDevInstance } from '@server/helpers/core-utils'
-import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { VIEW_LIFETIME } from '@server/initializers/constants'
-import { sendView } from '@server/lib/activitypub/send/send-view'
-import { PeerTubeSocket } from '@server/lib/peertube-socket'
-import { getServerActor } from '@server/models/application/application'
-import { VideoModel } from '@server/models/video/video'
-import { MVideo, MVideoImmutable } from '@server/types/models'
-import { buildUUID, sha256 } from '@shared/extra-utils'
-
-const lTags = loggerTagsFactory('views')
-
-export type ViewerScope = 'local' | 'remote'
-export type VideoScope = 'local' | 'remote'
-
-type Viewer = {
-  expires: number
-  id: string
-  viewerScope: ViewerScope
-  videoScope: VideoScope
-  lastFederation?: number
-}
-
-export class VideoViewerCounters {
-
-  // expires is new Date().getTime()
-  private readonly viewersPerVideo = new Map<number, Viewer[]>()
-  private readonly idToViewer = new Map<string, Viewer>()
-
-  private readonly salt = buildUUID()
-
-  private processingViewerCounters = false
-
-  constructor () {
-    setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER)
-  }
-
-  // ---------------------------------------------------------------------------
-
-  async addLocalViewer (options: {
-    video: MVideoImmutable
-    ip: string
-  }) {
-    const { video, ip } = options
-
-    logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) })
-
-    const viewerId = this.generateViewerId(ip, video.uuid)
-    const viewer = this.idToViewer.get(viewerId)
-
-    if (viewer) {
-      viewer.expires = this.buildViewerExpireTime()
-      await this.federateViewerIfNeeded(video, viewer)
-
-      return false
-    }
-
-    const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' })
-    await this.federateViewerIfNeeded(video, newViewer)
-
-    return true
-  }
-
-  async addRemoteViewer (options: {
-    video: MVideo
-    viewerId: string
-    viewerExpires: Date
-  }) {
-    const { video, viewerExpires, viewerId } = options
-
-    logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) })
-
-    await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' })
-
-    return true
-  }
-
-  // ---------------------------------------------------------------------------
-
-  getTotalViewers (options: {
-    viewerScope: ViewerScope
-    videoScope: VideoScope
-  }) {
-    let total = 0
-
-    for (const viewers of this.viewersPerVideo.values()) {
-      total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length
-    }
-
-    return total
-  }
-
-  getViewers (video: MVideo) {
-    const viewers = this.viewersPerVideo.get(video.id)
-    if (!viewers) return 0
-
-    return viewers.length
-  }
-
-  buildViewerExpireTime () {
-    return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER
-  }
-
-  // ---------------------------------------------------------------------------
-
-  private async addViewerToVideo (options: {
-    video: MVideoImmutable
-    viewerId: string
-    viewerScope: ViewerScope
-    viewerExpires?: Date
-  }) {
-    const { video, viewerExpires, viewerId, viewerScope } = options
-
-    let watchers = this.viewersPerVideo.get(video.id)
-
-    if (!watchers) {
-      watchers = []
-      this.viewersPerVideo.set(video.id, watchers)
-    }
-
-    const expires = viewerExpires
-      ? viewerExpires.getTime()
-      : this.buildViewerExpireTime()
-
-    const videoScope: VideoScope = video.remote
-      ? 'remote'
-      : 'local'
-
-    const viewer = { id: viewerId, expires, videoScope, viewerScope }
-    watchers.push(viewer)
-
-    this.idToViewer.set(viewerId, viewer)
-
-    await this.notifyClients(video.id, watchers.length)
-
-    return viewer
-  }
-
-  private async cleanViewerCounters () {
-    if (this.processingViewerCounters) return
-    this.processingViewerCounters = true
-
-    if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags())
-
-    try {
-      for (const videoId of this.viewersPerVideo.keys()) {
-        const notBefore = new Date().getTime()
-
-        const viewers = this.viewersPerVideo.get(videoId)
-
-        // Only keep not expired viewers
-        const newViewers: Viewer[] = []
-
-        // Filter new viewers
-        for (const viewer of viewers) {
-          if (viewer.expires > notBefore) {
-            newViewers.push(viewer)
-          } else {
-            this.idToViewer.delete(viewer.id)
-          }
-        }
-
-        if (newViewers.length === 0) this.viewersPerVideo.delete(videoId)
-        else this.viewersPerVideo.set(videoId, newViewers)
-
-        await this.notifyClients(videoId, newViewers.length)
-      }
-    } catch (err) {
-      logger.error('Error in video clean viewers scheduler.', { err, ...lTags() })
-    }
-
-    this.processingViewerCounters = false
-  }
-
-  private async notifyClients (videoId: string | number, viewersLength: number) {
-    const video = await VideoModel.loadImmutableAttributes(videoId)
-    if (!video) return
-
-    PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength)
-
-    logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags())
-  }
-
-  private generateViewerId (ip: string, videoUUID: string) {
-    return sha256(this.salt + '-' + ip + '-' + videoUUID)
-  }
-
-  private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) {
-    // Federate the viewer if it's been a "long" time we did not
-    const now = new Date().getTime()
-    const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75)
-
-    if (viewer.lastFederation && viewer.lastFederation > federationLimit) return
-
-    await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id })
-    viewer.lastFederation = now
-  }
-}
diff --git a/server/lib/views/shared/video-viewer-stats.ts b/server/lib/views/shared/video-viewer-stats.ts
deleted file mode 100644
index ebd963e59..000000000
--- a/server/lib/views/shared/video-viewer-stats.ts
+++ /dev/null
@@ -1,196 +0,0 @@
-import { Transaction } from 'sequelize/types'
-import { isTestOrDevInstance } from '@server/helpers/core-utils'
-import { GeoIP } from '@server/helpers/geo-ip'
-import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants'
-import { sequelizeTypescript } from '@server/initializers/database'
-import { sendCreateWatchAction } from '@server/lib/activitypub/send'
-import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url'
-import { Redis } from '@server/lib/redis'
-import { VideoModel } from '@server/models/video/video'
-import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
-import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section'
-import { MVideo, MVideoImmutable } from '@server/types/models'
-import { VideoViewEvent } from '@shared/models'
-
-const lTags = loggerTagsFactory('views')
-
-type LocalViewerStats = {
-  firstUpdated: number // Date.getTime()
-  lastUpdated: number // Date.getTime()
-
-  watchSections: {
-    start: number
-    end: number
-  }[]
-
-  watchTime: number
-
-  country: string
-
-  videoId: number
-}
-
-export class VideoViewerStats {
-  private processingViewersStats = false
-
-  constructor () {
-    setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS)
-  }
-
-  // ---------------------------------------------------------------------------
-
-  async addLocalViewer (options: {
-    video: MVideoImmutable
-    currentTime: number
-    ip: string
-    viewEvent?: VideoViewEvent
-  }) {
-    const { video, ip, viewEvent, currentTime } = options
-
-    logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) })
-
-    return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip })
-  }
-
-  // ---------------------------------------------------------------------------
-
-  async getWatchTime (videoId: number, ip: string) {
-    const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId })
-
-    return stats?.watchTime || 0
-  }
-
-  // ---------------------------------------------------------------------------
-
-  private async updateLocalViewerStats (options: {
-    video: MVideoImmutable
-    ip: string
-    currentTime: number
-    viewEvent?: VideoViewEvent
-  }) {
-    const { video, ip, viewEvent, currentTime } = options
-    const nowMs = new Date().getTime()
-
-    let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id })
-
-    if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) {
-      logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) })
-      return
-    }
-
-    if (!stats) {
-      const country = await GeoIP.Instance.safeCountryISOLookup(ip)
-
-      stats = {
-        firstUpdated: nowMs,
-        lastUpdated: nowMs,
-
-        watchSections: [],
-
-        watchTime: 0,
-
-        country,
-        videoId: video.id
-      }
-    }
-
-    stats.lastUpdated = nowMs
-
-    if (viewEvent === 'seek' || stats.watchSections.length === 0) {
-      stats.watchSections.push({
-        start: currentTime,
-        end: currentTime
-      })
-    } else {
-      const lastSection = stats.watchSections[stats.watchSections.length - 1]
-
-      if (lastSection.start > currentTime) {
-        logger.debug('Invalid end watch section %d. Last start record was at %d. Starting a new section.', currentTime, lastSection.start)
-
-        stats.watchSections.push({
-          start: currentTime,
-          end: currentTime
-        })
-      } else {
-        lastSection.end = currentTime
-      }
-    }
-
-    stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections)
-
-    logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) })
-
-    await Redis.Instance.setLocalVideoViewer(ip, video.id, stats)
-  }
-
-  async processViewerStats () {
-    if (this.processingViewersStats) return
-    this.processingViewersStats = true
-
-    if (!isTestOrDevInstance()) logger.info('Processing viewer statistics.', lTags())
-
-    const now = new Date().getTime()
-
-    try {
-      const allKeys = await Redis.Instance.listLocalVideoViewerKeys()
-
-      for (const key of allKeys) {
-        const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key })
-
-        // Process expired stats
-        if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) {
-          continue
-        }
-
-        try {
-          await sequelizeTypescript.transaction(async t => {
-            const video = await VideoModel.load(stats.videoId, t)
-            if (!video) return
-
-            const statsModel = await this.saveViewerStats(video, stats, t)
-
-            if (video.remote) {
-              await sendCreateWatchAction(statsModel, t)
-            }
-          })
-
-          await Redis.Instance.deleteLocalVideoViewersKeys(key)
-        } catch (err) {
-          logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() })
-        }
-      }
-    } catch (err) {
-      logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() })
-    }
-
-    this.processingViewersStats = false
-  }
-
-  private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) {
-    const statsModel = new LocalVideoViewerModel({
-      startDate: new Date(stats.firstUpdated),
-      endDate: new Date(stats.lastUpdated),
-      watchTime: stats.watchTime,
-      country: stats.country,
-      videoId: video.id
-    })
-
-    statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel)
-    statsModel.Video = video as VideoModel
-
-    await statsModel.save({ transaction })
-
-    statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({
-      localVideoViewerId: statsModel.id,
-      watchSections: stats.watchSections,
-      transaction
-    })
-
-    return statsModel
-  }
-
-  private buildWatchTimeFromSections (sections: { start: number, end: number }[]) {
-    return sections.reduce((p, current) => p + (current.end - current.start), 0)
-  }
-}
diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts
deleted file mode 100644
index e563287e1..000000000
--- a/server/lib/views/shared/video-views.ts
+++ /dev/null
@@ -1,70 +0,0 @@
-import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { sendView } from '@server/lib/activitypub/send/send-view'
-import { getCachedVideoDuration } from '@server/lib/video'
-import { getServerActor } from '@server/models/application/application'
-import { MVideo, MVideoImmutable } from '@server/types/models'
-import { buildUUID } from '@shared/extra-utils'
-import { Redis } from '../../redis'
-
-const lTags = loggerTagsFactory('views')
-
-export class VideoViews {
-
-  async addLocalView (options: {
-    video: MVideoImmutable
-    ip: string
-    watchTime: number
-  }) {
-    const { video, ip, watchTime } = options
-
-    logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) })
-
-    if (!await this.hasEnoughWatchTime(video, watchTime)) return false
-
-    const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid)
-    if (viewExists) return false
-
-    await Redis.Instance.setIPVideoView(ip, video.uuid)
-
-    await this.addView(video)
-
-    await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() })
-
-    return true
-  }
-
-  async addRemoteView (options: {
-    video: MVideo
-  }) {
-    const { video } = options
-
-    logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) })
-
-    await this.addView(video)
-
-    return true
-  }
-
-  // ---------------------------------------------------------------------------
-
-  private async addView (video: MVideoImmutable) {
-    const promises: Promise<any>[] = []
-
-    if (video.isOwned()) {
-      promises.push(Redis.Instance.addLocalVideoView(video.id))
-    }
-
-    promises.push(Redis.Instance.addVideoViewStats(video.id))
-
-    await Promise.all(promises)
-  }
-
-  private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) {
-    const { duration, isLive } = await getCachedVideoDuration(video.id)
-
-    if (isLive || duration >= 30) return watchTime >= 30
-
-    // Check more than 50% of the video is watched
-    return duration / watchTime < 2
-  }
-}
diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts
deleted file mode 100644
index c088dad5e..000000000
--- a/server/lib/views/video-views-manager.ts
+++ /dev/null
@@ -1,100 +0,0 @@
-import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { MVideo, MVideoImmutable } from '@server/types/models'
-import { VideoViewEvent } from '@shared/models'
-import { VideoScope, VideoViewerCounters, VideoViewerStats, VideoViews, ViewerScope } from './shared'
-
-/**
- * If processing a local view:
- *  - We update viewer information (segments watched, watch time etc)
- *  - We add +1 to video viewers counter if this is a new viewer
- *  - We add +1 to video views counter if this is a new view and if the user watched enough seconds
- *  - We send AP message to notify about this viewer and this view
- *  - We update last video time for the user if authenticated
- *
- * If processing a remote view:
- *  - We add +1 to video viewers counter
- *  - We add +1 to video views counter
- *
- * A viewer is a someone that watched one or multiple sections of a video
- * A viewer that watched only a few seconds of a video may not increment the video views counter
- * Viewers statistics are sent to origin instance using the `WatchAction` ActivityPub object
- *
- */
-
-const lTags = loggerTagsFactory('views')
-
-export class VideoViewsManager {
-
-  private static instance: VideoViewsManager
-
-  private videoViewerStats: VideoViewerStats
-  private videoViewerCounters: VideoViewerCounters
-  private videoViews: VideoViews
-
-  private constructor () {
-  }
-
-  init () {
-    this.videoViewerStats = new VideoViewerStats()
-    this.videoViewerCounters = new VideoViewerCounters()
-    this.videoViews = new VideoViews()
-  }
-
-  async processLocalView (options: {
-    video: MVideoImmutable
-    currentTime: number
-    ip: string | null
-    viewEvent?: VideoViewEvent
-  }) {
-    const { video, ip, viewEvent, currentTime } = options
-
-    logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags())
-
-    await this.videoViewerStats.addLocalViewer({ video, ip, viewEvent, currentTime })
-
-    const successViewer = await this.videoViewerCounters.addLocalViewer({ video, ip })
-
-    // Do it after added local viewer to fetch updated information
-    const watchTime = await this.videoViewerStats.getWatchTime(video.id, ip)
-
-    const successView = await this.videoViews.addLocalView({ video, watchTime, ip })
-
-    return { successView, successViewer }
-  }
-
-  async processRemoteView (options: {
-    video: MVideo
-    viewerId: string | null
-    viewerExpires?: Date
-  }) {
-    const { video, viewerId, viewerExpires } = options
-
-    logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() })
-
-    if (viewerExpires) await this.videoViewerCounters.addRemoteViewer({ video, viewerId, viewerExpires })
-    else await this.videoViews.addRemoteView({ video })
-  }
-
-  getViewers (video: MVideo) {
-    return this.videoViewerCounters.getViewers(video)
-  }
-
-  getTotalViewers (options: {
-    viewerScope: ViewerScope
-    videoScope: VideoScope
-  }) {
-    return this.videoViewerCounters.getTotalViewers(options)
-  }
-
-  buildViewerExpireTime () {
-    return this.videoViewerCounters.buildViewerExpireTime()
-  }
-
-  processViewerStats () {
-    return this.videoViewerStats.processViewerStats()
-  }
-
-  static get Instance () {
-    return this.instance || (this.instance = new this())
-  }
-}