Add playback metric endpoint sent to OTEL

12 files changed, 397 insertions, 13 deletions

diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts
index 8c8ebd061..e1d197c8a 100644
--- a/server/controllers/api/index.ts
+++ b/server/controllers/api/index.ts
@@ -11,6 +11,7 @@ import { bulkRouter } from './bulk'
 import { configRouter } from './config'
 import { customPageRouter } from './custom-page'
 import { jobsRouter } from './jobs'
+import { metricsRouter } from './metrics'
 import { oauthClientsRouter } from './oauth-clients'
 import { overviewsRouter } from './overviews'
 import { pluginRouter } from './plugins'
@@ -18,9 +19,9 @@ import { searchRouter } from './search'
 import { serverRouter } from './server'
 import { usersRouter } from './users'
 import { videoChannelRouter } from './video-channel'
+import { videoChannelSyncRouter } from './video-channel-sync'
 import { videoPlaylistRouter } from './video-playlist'
 import { videosRouter } from './videos'
-import { videoChannelSyncRouter } from './video-channel-sync'
 
 const apiRouter = express.Router()
 
@@ -48,6 +49,7 @@ apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
 apiRouter.use('/video-playlists', videoPlaylistRouter)
 apiRouter.use('/videos', videosRouter)
 apiRouter.use('/jobs', jobsRouter)
+apiRouter.use('/metrics', metricsRouter)
 apiRouter.use('/search', searchRouter)
 apiRouter.use('/overviews', overviewsRouter)
 apiRouter.use('/plugins', pluginRouter)
diff --git a/server/controllers/api/metrics.ts b/server/controllers/api/metrics.ts
new file mode 100644
index 000000000..578b023a1
--- /dev/null
+++ b/server/controllers/api/metrics.ts
@@ -0,0 +1,27 @@
+import express from 'express'
+import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
+import { HttpStatusCode, PlaybackMetricCreate } from '@shared/models'
+import { addPlaybackMetricValidator, asyncMiddleware } from '../../middlewares'
+
+const metricsRouter = express.Router()
+
+metricsRouter.post('/playback',
+  asyncMiddleware(addPlaybackMetricValidator),
+  addPlaybackMetric
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  metricsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+function addPlaybackMetric (req: express.Request, res: express.Response) {
+  const body: PlaybackMetricCreate = req.body
+
+  OpenTelemetryMetrics.Instance.observePlaybackMetric(res.locals.onlyImmutableVideo, body)
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
diff --git a/server/helpers/custom-validators/metrics.ts b/server/helpers/custom-validators/metrics.ts
new file mode 100644
index 000000000..533f8988d
--- /dev/null
+++ b/server/helpers/custom-validators/metrics.ts
@@ -0,0 +1,9 @@
+function isValidPlayerMode (value: any) {
+  return value === 'webtorrent' || value === 'p2p-media-loader'
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidPlayerMode
+}
diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts
index 1b3813743..775d954ba 100644
--- a/server/lib/opentelemetry/metric-helpers/index.ts
+++ b/server/lib/opentelemetry/metric-helpers/index.ts
@@ -1,5 +1,6 @@
 export * from './lives-observers-builder'
 export * from './job-queue-observers-builder'
 export * from './nodejs-observers-builder'
+export * from './playback-metrics'
 export * from './stats-observers-builder'
 export * from './viewers-observers-builder'
diff --git a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts
index 766cbe03b..473015e91 100644
--- a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts
+++ b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts
@@ -2,7 +2,7 @@ import { readdir } from 'fs-extra'
 import { constants, PerformanceObserver } from 'perf_hooks'
 import * as process from 'process'
 import { Meter, ObservableResult } from '@opentelemetry/api-metrics'
-import { ExplicitBucketHistogramAggregation, MeterProvider } from '@opentelemetry/sdk-metrics-base'
+import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics-base'
 import { View } from '@opentelemetry/sdk-metrics-base/build/src/view/View'
 import { logger } from '@server/helpers/logger'
 
@@ -12,7 +12,16 @@ import { logger } from '@server/helpers/logger'
 
 export class NodeJSObserversBuilder {
 
-  constructor (private readonly meter: Meter, private readonly meterProvider: MeterProvider) {
+  constructor (private readonly meter: Meter) {
+  }
+
+  static getViews () {
+    return [
+      new View({
+        aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]),
+        instrumentName: 'nodejs_gc_duration_seconds'
+      })
+    ]
   }
 
   buildObservers () {
@@ -91,11 +100,6 @@ export class NodeJSObserversBuilder {
       [constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb'
     }
 
-    this.meterProvider.addView(
-      new View({ aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]) }),
-      { instrument: { name: 'nodejs_gc_duration_seconds' } }
-    )
-
     const histogram = this.meter.createHistogram('nodejs_gc_duration_seconds', {
       description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb'
     })
diff --git a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts
new file mode 100644
index 000000000..d2abdee62
--- /dev/null
+++ b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts
@@ -0,0 +1,59 @@
+import { Counter, Meter } from '@opentelemetry/api-metrics'
+import { MVideoImmutable } from '@server/types/models'
+import { PlaybackMetricCreate } from '@shared/models'
+
+export class PlaybackMetrics {
+  private errorsCounter: Counter
+  private resolutionChangesCounter: Counter
+
+  private downloadedBytesP2PCounter: Counter
+  private uploadedBytesP2PCounter: Counter
+
+  private downloadedBytesHTTPCounter: Counter
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildCounters () {
+    this.errorsCounter = this.meter.createCounter('peertube_playback_errors_count', {
+      description: 'Errors collected from PeerTube player.'
+    })
+
+    this.resolutionChangesCounter = this.meter.createCounter('peertube_playback_resolution_changes_count', {
+      description: 'Resolution changes collected from PeerTube player.'
+    })
+
+    this.downloadedBytesHTTPCounter = this.meter.createCounter('peertube_playback_http_downloaded_bytes', {
+      description: 'Downloaded bytes with HTTP by PeerTube player.'
+    })
+    this.downloadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_downloaded_bytes', {
+      description: 'Downloaded bytes with P2P by PeerTube player.'
+    })
+
+    this.uploadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_uploaded_bytes', {
+      description: 'Uploaded bytes with P2P by PeerTube player.'
+    })
+  }
+
+  observe (video: MVideoImmutable, metrics: PlaybackMetricCreate) {
+    const attributes = {
+      videoOrigin: video.remote
+        ? 'remote'
+        : 'local',
+
+      playerMode: metrics.playerMode,
+
+      resolution: metrics.resolution + '',
+      fps: metrics.fps + ''
+    }
+
+    this.errorsCounter.add(metrics.errors, attributes)
+    this.resolutionChangesCounter.add(metrics.resolutionChanges, attributes)
+
+    this.downloadedBytesHTTPCounter.add(metrics.downloadedBytesHTTP, attributes)
+    this.downloadedBytesP2PCounter.add(metrics.downloadedBytesP2P, attributes)
+
+    this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes)
+  }
+}
diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts
index ffe493670..ba33c9505 100644
--- a/server/lib/opentelemetry/metrics.ts
+++ b/server/lib/opentelemetry/metrics.ts
@@ -4,10 +4,13 @@ import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
 import { MeterProvider } from '@opentelemetry/sdk-metrics-base'
 import { logger } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
+import { MVideoImmutable } from '@server/types/models'
+import { PlaybackMetricCreate } from '@shared/models'
 import {
   JobQueueObserversBuilder,
   LivesObserversBuilder,
   NodeJSObserversBuilder,
+  PlaybackMetrics,
   StatsObserversBuilder,
   ViewersObserversBuilder
 } from './metric-helpers'
@@ -20,6 +23,8 @@ class OpenTelemetryMetrics {
 
   private onRequestDuration: (req: Request, res: Response) => void
 
+  private playbackMetrics: PlaybackMetrics
+
   private constructor () {}
 
   init (app: Application) {
@@ -41,7 +46,11 @@ class OpenTelemetryMetrics {
 
     logger.info('Registering Open Telemetry metrics')
 
-    const provider = new MeterProvider()
+    const provider = new MeterProvider({
+      views: [
+        ...NodeJSObserversBuilder.getViews()
+      ]
+    })
 
     provider.addMetricReader(new PrometheusExporter({ port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT }))
 
@@ -51,7 +60,10 @@ class OpenTelemetryMetrics {
 
     this.buildRequestObserver()
 
-    const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter, provider)
+    this.playbackMetrics = new PlaybackMetrics(this.meter)
+    this.playbackMetrics.buildCounters()
+
+    const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter)
     nodeJSObserversBuilder.buildObservers()
 
     const jobQueueObserversBuilder = new JobQueueObserversBuilder(this.meter)
@@ -67,6 +79,10 @@ class OpenTelemetryMetrics {
     viewersObserversBuilder.buildObservers()
   }
 
+  observePlaybackMetric (video: MVideoImmutable, metrics: PlaybackMetricCreate) {
+    this.playbackMetrics.observe(video, metrics)
+  }
+
   private buildRequestObserver () {
     const requestDuration = this.meter.createHistogram('http_request_duration_ms', {
       unit: 'milliseconds',
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts
index b0ad04819..ffadb3b49 100644
--- a/server/middlewares/validators/index.ts
+++ b/server/middlewares/validators/index.ts
@@ -10,6 +10,7 @@ export * from './express'
 export * from './feeds'
 export * from './follows'
 export * from './jobs'
+export * from './metrics'
 export * from './logs'
 export * from './oembed'
 export * from './pagination'
diff --git a/server/middlewares/validators/metrics.ts b/server/middlewares/validators/metrics.ts
new file mode 100644
index 000000000..b1dbec603
--- /dev/null
+++ b/server/middlewares/validators/metrics.ts
@@ -0,0 +1,56 @@
+import express from 'express'
+import { body } from 'express-validator'
+import { isValidPlayerMode } from '@server/helpers/custom-validators/metrics'
+import { isIdOrUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc'
+import { CONFIG } from '@server/initializers/config'
+import { HttpStatusCode, PlaybackMetricCreate } from '@shared/models'
+import { logger } from '../../helpers/logger'
+import { areValidationErrors, doesVideoExist } from './shared'
+
+const addPlaybackMetricValidator = [
+  body('resolution')
+    .isInt({ min: 0 }).withMessage('Invalid resolution'),
+  body('fps')
+    .optional()
+    .isInt({ min: 0 }).withMessage('Invalid fps'),
+  body('playerMode')
+    .custom(isValidPlayerMode).withMessage('Invalid playerMode'),
+
+  body('resolutionChanges')
+    .isInt({ min: 0 }).withMessage('Invalid resolutionChanges'),
+
+  body('errors')
+    .isInt({ min: 0 }).withMessage('Invalid errors'),
+
+  body('downloadedBytesP2P')
+    .isInt({ min: 0 }).withMessage('Invalid downloadedBytesP2P'),
+  body('downloadedBytesHTTP')
+    .isInt({ min: 0 }).withMessage('Invalid downloadedBytesHTTP'),
+
+  body('uploadedBytesP2P')
+    .isInt({ min: 0 }).withMessage('Invalid uploadedBytesP2P'),
+
+  body('videoId')
+    .customSanitizer(toCompleteUUID)
+    .optional()
+    .custom(isIdOrUUIDValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking addPlaybackMetricValidator parameters.', { parameters: req.query })
+
+    if (!CONFIG.OPEN_TELEMETRY.METRICS.ENABLED) return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+
+    const body: PlaybackMetricCreate = req.body
+
+    if (areValidationErrors(req, res)) return
+    if (!await doesVideoExist(body.videoId, res, 'only-immutable-attributes')) return
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  addPlaybackMetricValidator
+}
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 149305f49..cd7a38459 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -10,6 +10,7 @@ import './follows'
 import './jobs'
 import './live'
 import './logs'
+import './metrics'
 import './my-user'
 import './plugins'
 import './redundancy'
diff --git a/server/tests/api/check-params/metrics.ts b/server/tests/api/check-params/metrics.ts
new file mode 100644
index 000000000..2d4509406
--- /dev/null
+++ b/server/tests/api/check-params/metrics.ts
@@ -0,0 +1,183 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import 'mocha'
+import { omit } from 'lodash'
+import { HttpStatusCode, PlaybackMetricCreate, VideoResolution } from '@shared/models'
+import { cleanupTests, createSingleServer, makePostBodyRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
+
+describe('Test metrics API validators', function () {
+  let server: PeerTubeServer
+  let videoUUID: string
+
+  // ---------------------------------------------------------------
+
+  before(async function () {
+    this.timeout(120000)
+
+    server = await createSingleServer(1, {
+      open_telemetry: {
+        metrics: {
+          enabled: true
+        }
+      }
+    })
+
+    await setAccessTokensToServers([ server ])
+
+    const { uuid } = await server.videos.quickUpload({ name: 'video' })
+    videoUUID = uuid
+  })
+
+  describe('When adding playback metrics', function () {
+    const path = '/api/v1/metrics/playback'
+    let baseParams: PlaybackMetricCreate
+
+    before(function () {
+      baseParams = {
+        playerMode: 'p2p-media-loader',
+        resolution: VideoResolution.H_1080P,
+        fps: 30,
+        resolutionChanges: 1,
+        errors: 2,
+        downloadedBytesP2P: 0,
+        downloadedBytesHTTP: 0,
+        uploadedBytesP2P: 0,
+        videoId: videoUUID
+      }
+    })
+
+    it('Should fail with an invalid resolution', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, resolution: 'toto' }
+      })
+    })
+
+    it('Should fail with an invalid fps', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, fps: 'toto' }
+      })
+    })
+
+    it('Should fail with a missing/invalid player mode', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: omit(baseParams, 'playerMode')
+      })
+
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, playerMode: 'toto' }
+      })
+    })
+
+    it('Should fail with an missing/invalid resolution changes', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: omit(baseParams, 'resolutionChanges')
+      })
+
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, resolutionChanges: 'toto' }
+      })
+    })
+
+    it('Should fail with a missing errors', async function () {
+
+    })
+
+    it('Should fail with an missing/invalid errors', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: omit(baseParams, 'errors')
+      })
+
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, errors: 'toto' }
+      })
+    })
+
+    it('Should fail with an missing/invalid downloadedBytesP2P', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: omit(baseParams, 'downloadedBytesP2P')
+      })
+
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, downloadedBytesP2P: 'toto' }
+      })
+    })
+
+    it('Should fail with an missing/invalid downloadedBytesHTTP', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: omit(baseParams, 'downloadedBytesHTTP')
+      })
+
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, downloadedBytesHTTP: 'toto' }
+      })
+    })
+
+    it('Should fail with an missing/invalid uploadedBytesP2P', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: omit(baseParams, 'uploadedBytesP2P')
+      })
+
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, uploadedBytesP2P: 'toto' }
+      })
+    })
+
+    it('Should fail with a bad video id', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, videoId: 'toto' }
+      })
+    })
+
+    it('Should fail with an unknown video', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: { ...baseParams, videoId: 42 },
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should succeed with the correct params', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        fields: baseParams,
+        expectedStatus: HttpStatusCode.NO_CONTENT_204
+      })
+    })
+  })
+
+  after(async function () {
+    await cleanupTests([ server ])
+  })
+})
diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts
index 20909429f..3137a9eb6 100644
--- a/server/tests/api/server/open-telemetry.ts
+++ b/server/tests/api/server/open-telemetry.ts
@@ -2,14 +2,14 @@
 
 import { expect } from 'chai'
 import { expectLogContain, expectLogDoesNotContain, MockHTTP } from '@server/tests/shared'
-import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import { HttpStatusCode, VideoPrivacy, VideoResolution } from '@shared/models'
 import { cleanupTests, createSingleServer, makeRawRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
 
 describe('Open Telemetry', function () {
   let server: PeerTubeServer
 
   describe('Metrics', function () {
-    const metricsUrl = 'http://localhost:9091/metrics'
+    const metricsUrl = 'http://localhost:9092/metrics'
 
     it('Should not enable open telemetry metrics', async function () {
       server = await createSingleServer(1)
@@ -36,8 +36,33 @@ describe('Open Telemetry', function () {
       })
 
       const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200)
-      expect(res.text).to.contain('peertube_job_queue_total')
+      expect(res.text).to.contain('peertube_job_queue_total{')
+    })
+
+    it('Should have playback metrics', async function () {
+      await setAccessTokensToServers([ server ])
+
+      const video = await server.videos.quickUpload({ name: 'video' })
+
+      await server.metrics.addPlaybackMetric({
+        metrics: {
+          playerMode: 'p2p-media-loader',
+          resolution: VideoResolution.H_1080P,
+          fps: 30,
+          resolutionChanges: 1,
+          errors: 2,
+          downloadedBytesP2P: 0,
+          downloadedBytesHTTP: 0,
+          uploadedBytesP2P: 5,
+          videoId: video.uuid
+        }
+      })
 
+      const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200)
+      expect(res.text).to.contain('peertube_playback_http_uploaded_bytes_total{')
+    })
+
+    after(async function () {
       await server.kill()
     })
   })