Add live and viewers otel metrics

8 files changed, 106 insertions, 14 deletions

diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts
index ff0aff9fd..1b3813743 100644
--- a/server/lib/opentelemetry/metric-helpers/index.ts
+++ b/server/lib/opentelemetry/metric-helpers/index.ts
@@ -1,3 +1,5 @@
+export * from './lives-observers-builder'
 export * from './job-queue-observers-builder'
 export * from './nodejs-observers-builder'
 export * from './stats-observers-builder'
+export * from './viewers-observers-builder'
diff --git a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts
new file mode 100644
index 000000000..e27bd8548
--- /dev/null
+++ b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts
@@ -0,0 +1,21 @@
+import { Meter } from '@opentelemetry/api-metrics'
+import { VideoModel } from '@server/models/video/video'
+
+export class LivesObserversBuilder {
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildObservers () {
+    this.meter.createObservableGauge('peertube_running_lives_total', {
+      description: 'Total running lives on the instance'
+    }).addCallback(async observableResult => {
+      const local = await VideoModel.countLives({ remote: false, mode: 'published' })
+      const remote = await VideoModel.countLives({ remote: true, mode: 'published' })
+
+      observableResult.observe(local, { liveOrigin: 'local' })
+      observableResult.observe(remote, { liveOrigin: 'remote' })
+    })
+  }
+}
diff --git a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts
new file mode 100644
index 000000000..634e5bbc9
--- /dev/null
+++ b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts
@@ -0,0 +1,24 @@
+import { Meter } from '@opentelemetry/api-metrics'
+import { VideoScope, ViewerScope } from '@server/lib/views/shared'
+import { VideoViewsManager } from '@server/lib/views/video-views-manager'
+
+export class ViewersObserversBuilder {
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildObservers () {
+    this.meter.createObservableGauge('peertube_viewers_total', {
+      description: 'Total viewers on the instance'
+    }).addCallback(observableResult => {
+      for (const viewerScope of [ 'local', 'remote' ] as ViewerScope[]) {
+        for (const videoScope of [ 'local', 'remote' ] as VideoScope[]) {
+          const result = VideoViewsManager.Instance.getTotalViewers({ viewerScope, videoScope })
+
+          observableResult.observe(result, { viewerOrigin: viewerScope, videoOrigin: videoScope })
+        }
+      }
+    })
+  }
+}
diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts
index 149f421be..ffe493670 100644
--- a/server/lib/opentelemetry/metrics.ts
+++ b/server/lib/opentelemetry/metrics.ts
@@ -4,7 +4,13 @@ import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
 import { MeterProvider } from '@opentelemetry/sdk-metrics-base'
 import { logger } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
-import { JobQueueObserversBuilder, NodeJSObserversBuilder, StatsObserversBuilder } from './metric-helpers'
+import {
+  JobQueueObserversBuilder,
+  LivesObserversBuilder,
+  NodeJSObserversBuilder,
+  StatsObserversBuilder,
+  ViewersObserversBuilder
+} from './metric-helpers'
 
 class OpenTelemetryMetrics {
 
@@ -53,6 +59,12 @@ class OpenTelemetryMetrics {
 
     const statsObserversBuilder = new StatsObserversBuilder(this.meter)
     statsObserversBuilder.buildObservers()
+
+    const livesObserversBuilder = new LivesObserversBuilder(this.meter)
+    livesObserversBuilder.buildObservers()
+
+    const viewersObserversBuilder = new ViewersObserversBuilder(this.meter)
+    viewersObserversBuilder.buildObservers()
   }
 
   private buildRequestObserver () {
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts
index f851ce050..f5b83130e 100644
--- a/server/lib/views/shared/video-viewer-counters.ts
+++ b/server/lib/views/shared/video-viewer-counters.ts
@@ -10,9 +10,14 @@ import { buildUUID, sha256 } from '@shared/extra-utils'
 
 const lTags = loggerTagsFactory('views')
 
+export type ViewerScope = 'local' | 'remote'
+export type VideoScope = 'local' | 'remote'
+
 type Viewer = {
   expires: number
   id: string
+  viewerScope: ViewerScope
+  videoScope: VideoScope
   lastFederation?: number
 }
 
@@ -50,7 +55,7 @@ export class VideoViewerCounters {
       return false
     }
 
-    const newViewer = await this.addViewerToVideo({ viewerId, video })
+    const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' })
     await this.federateViewerIfNeeded(video, newViewer)
 
     return true
@@ -65,13 +70,26 @@ export class VideoViewerCounters {
 
     logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) })
 
-    await this.addViewerToVideo({ video, viewerExpires, viewerId })
+    await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' })
 
     return true
   }
 
   // ---------------------------------------------------------------------------
 
+  getTotalViewers (options: {
+    viewerScope: ViewerScope
+    videoScope: VideoScope
+  }) {
+    let total = 0
+
+    for (const viewers of this.viewersPerVideo.values()) {
+      total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length
+    }
+
+    return total
+  }
+
   getViewers (video: MVideo) {
     const viewers = this.viewersPerVideo.get(video.id)
     if (!viewers) return 0
@@ -88,9 +106,10 @@ export class VideoViewerCounters {
   private async addViewerToVideo (options: {
     video: MVideoImmutable
     viewerId: string
+    viewerScope: ViewerScope
     viewerExpires?: Date
   }) {
-    const { video, viewerExpires, viewerId } = options
+    const { video, viewerExpires, viewerId, viewerScope } = options
 
     let watchers = this.viewersPerVideo.get(video.id)
 
@@ -103,7 +122,11 @@ export class VideoViewerCounters {
       ? viewerExpires.getTime()
       : this.buildViewerExpireTime()
 
-    const viewer = { id: viewerId, expires }
+    const videoScope: VideoScope = video.remote
+      ? 'remote'
+      : 'local'
+
+    const viewer = { id: viewerId, expires, videoScope, viewerScope }
     watchers.push(viewer)
 
     this.idToViewer.set(viewerId, viewer)
diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts
index 86758e8d8..c088dad5e 100644
--- a/server/lib/views/video-views-manager.ts
+++ b/server/lib/views/video-views-manager.ts
@@ -1,7 +1,7 @@
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { MVideo, MVideoImmutable } from '@server/types/models'
 import { VideoViewEvent } from '@shared/models'
-import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared'
+import { VideoScope, VideoViewerCounters, VideoViewerStats, VideoViews, ViewerScope } from './shared'
 
 /**
  * If processing a local view:
@@ -79,6 +79,13 @@ export class VideoViewsManager {
     return this.videoViewerCounters.getViewers(video)
   }
 
+  getTotalViewers (options: {
+    viewerScope: ViewerScope
+    videoScope: VideoScope
+  }) {
+    return this.videoViewerCounters.getTotalViewers(options)
+  }
+
   buildViewerExpireTime () {
     return this.videoViewerCounters.buildViewerExpireTime()
   }
diff --git a/server/middlewares/validators/videos/video-live.ts b/server/middlewares/validators/videos/video-live.ts
index 59638d5e0..777b57e9a 100644
--- a/server/middlewares/validators/videos/video-live.ts
+++ b/server/middlewares/validators/videos/video-live.ts
@@ -119,7 +119,7 @@ const videoLiveAddValidator = getCommonVideoEditAttributes().concat([
     if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req)
 
     if (CONFIG.LIVE.MAX_INSTANCE_LIVES !== -1) {
-      const totalInstanceLives = await VideoModel.countLocalLives()
+      const totalInstanceLives = await VideoModel.countLives({ remote: false, mode: 'not-ended' })
 
       if (totalInstanceLives >= CONFIG.LIVE.MAX_INSTANCE_LIVES) {
         cleanUpReqFiles(req)
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 27e605be6..924f12a5e 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1209,18 +1209,21 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
     return VideoModel.getAvailableForApi(queryOptions)
   }
 
-  static countLocalLives () {
-    const options = {
+  static countLives (options: {
+    remote: boolean
+    mode: 'published' | 'not-ended'
+  }) {
+    const query = {
       where: {
-        remote: false,
+        remote: options.remote,
         isLive: true,
-        state: {
-          [Op.ne]: VideoState.LIVE_ENDED
-        }
+        state: options.mode === 'not-ended'
+          ? { [Op.ne]: VideoState.LIVE_ENDED }
+          : { [Op.eq]: VideoState.PUBLISHED }
       }
     }
 
-    return VideoModel.count(options)
+    return VideoModel.count(query)
   }
 
   static countVideosUploadedByUserSince (userId: number, since: Date) {