From fd3c2e87051f5029cdec39d877b576a62f48e219 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 12 Aug 2022 16:41:29 +0200 Subject: Add playback metric endpoint sent to OTEL --- server/controllers/api/index.ts | 4 +- server/controllers/api/metrics.ts | 27 +++ server/helpers/custom-validators/metrics.ts | 9 + server/lib/opentelemetry/metric-helpers/index.ts | 1 + .../metric-helpers/nodejs-observers-builder.ts | 18 +- .../metric-helpers/playback-metrics.ts | 59 +++++++ server/lib/opentelemetry/metrics.ts | 20 ++- server/middlewares/validators/index.ts | 1 + server/middlewares/validators/metrics.ts | 56 +++++++ server/tests/api/check-params/index.ts | 1 + server/tests/api/check-params/metrics.ts | 183 +++++++++++++++++++++ server/tests/api/server/open-telemetry.ts | 31 +++- 12 files changed, 397 insertions(+), 13 deletions(-) create mode 100644 server/controllers/api/metrics.ts create mode 100644 server/helpers/custom-validators/metrics.ts create mode 100644 server/lib/opentelemetry/metric-helpers/playback-metrics.ts create mode 100644 server/middlewares/validators/metrics.ts create mode 100644 server/tests/api/check-params/metrics.ts (limited to 'server') diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index 8c8ebd061..e1d197c8a 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts @@ -11,6 +11,7 @@ import { bulkRouter } from './bulk' import { configRouter } from './config' import { customPageRouter } from './custom-page' import { jobsRouter } from './jobs' +import { metricsRouter } from './metrics' import { oauthClientsRouter } from './oauth-clients' import { overviewsRouter } from './overviews' import { pluginRouter } from './plugins' @@ -18,9 +19,9 @@ import { searchRouter } from './search' import { serverRouter } from './server' import { usersRouter } from './users' import { videoChannelRouter } from './video-channel' +import { videoChannelSyncRouter } from './video-channel-sync' import { videoPlaylistRouter } from './video-playlist' import { videosRouter } from './videos' -import { videoChannelSyncRouter } from './video-channel-sync' const apiRouter = express.Router() @@ -48,6 +49,7 @@ apiRouter.use('/video-channel-syncs', videoChannelSyncRouter) apiRouter.use('/video-playlists', videoPlaylistRouter) apiRouter.use('/videos', videosRouter) apiRouter.use('/jobs', jobsRouter) +apiRouter.use('/metrics', metricsRouter) apiRouter.use('/search', searchRouter) apiRouter.use('/overviews', overviewsRouter) apiRouter.use('/plugins', pluginRouter) diff --git a/server/controllers/api/metrics.ts b/server/controllers/api/metrics.ts new file mode 100644 index 000000000..578b023a1 --- /dev/null +++ b/server/controllers/api/metrics.ts @@ -0,0 +1,27 @@ +import express from 'express' +import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics' +import { HttpStatusCode, PlaybackMetricCreate } from '@shared/models' +import { addPlaybackMetricValidator, asyncMiddleware } from '../../middlewares' + +const metricsRouter = express.Router() + +metricsRouter.post('/playback', + asyncMiddleware(addPlaybackMetricValidator), + addPlaybackMetric +) + +// --------------------------------------------------------------------------- + +export { + metricsRouter +} + +// --------------------------------------------------------------------------- + +function addPlaybackMetric (req: express.Request, res: express.Response) { + const body: PlaybackMetricCreate = req.body + + OpenTelemetryMetrics.Instance.observePlaybackMetric(res.locals.onlyImmutableVideo, body) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/helpers/custom-validators/metrics.ts b/server/helpers/custom-validators/metrics.ts new file mode 100644 index 000000000..533f8988d --- /dev/null +++ b/server/helpers/custom-validators/metrics.ts @@ -0,0 +1,9 @@ +function isValidPlayerMode (value: any) { + return value === 'webtorrent' || value === 'p2p-media-loader' +} + +// --------------------------------------------------------------------------- + +export { + isValidPlayerMode +} diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts index 1b3813743..775d954ba 100644 --- a/server/lib/opentelemetry/metric-helpers/index.ts +++ b/server/lib/opentelemetry/metric-helpers/index.ts @@ -1,5 +1,6 @@ export * from './lives-observers-builder' export * from './job-queue-observers-builder' export * from './nodejs-observers-builder' +export * from './playback-metrics' export * from './stats-observers-builder' export * from './viewers-observers-builder' diff --git a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts index 766cbe03b..473015e91 100644 --- a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts +++ b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts @@ -2,7 +2,7 @@ import { readdir } from 'fs-extra' import { constants, PerformanceObserver } from 'perf_hooks' import * as process from 'process' import { Meter, ObservableResult } from '@opentelemetry/api-metrics' -import { ExplicitBucketHistogramAggregation, MeterProvider } from '@opentelemetry/sdk-metrics-base' +import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics-base' import { View } from '@opentelemetry/sdk-metrics-base/build/src/view/View' import { logger } from '@server/helpers/logger' @@ -12,7 +12,16 @@ import { logger } from '@server/helpers/logger' export class NodeJSObserversBuilder { - constructor (private readonly meter: Meter, private readonly meterProvider: MeterProvider) { + constructor (private readonly meter: Meter) { + } + + static getViews () { + return [ + new View({ + aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]), + instrumentName: 'nodejs_gc_duration_seconds' + }) + ] } buildObservers () { @@ -91,11 +100,6 @@ export class NodeJSObserversBuilder { [constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb' } - this.meterProvider.addView( - new View({ aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]) }), - { instrument: { name: 'nodejs_gc_duration_seconds' } } - ) - const histogram = this.meter.createHistogram('nodejs_gc_duration_seconds', { description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb' }) diff --git a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts new file mode 100644 index 000000000..d2abdee62 --- /dev/null +++ b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts @@ -0,0 +1,59 @@ +import { Counter, Meter } from '@opentelemetry/api-metrics' +import { MVideoImmutable } from '@server/types/models' +import { PlaybackMetricCreate } from '@shared/models' + +export class PlaybackMetrics { + private errorsCounter: Counter + private resolutionChangesCounter: Counter + + private downloadedBytesP2PCounter: Counter + private uploadedBytesP2PCounter: Counter + + private downloadedBytesHTTPCounter: Counter + + constructor (private readonly meter: Meter) { + + } + + buildCounters () { + this.errorsCounter = this.meter.createCounter('peertube_playback_errors_count', { + description: 'Errors collected from PeerTube player.' + }) + + this.resolutionChangesCounter = this.meter.createCounter('peertube_playback_resolution_changes_count', { + description: 'Resolution changes collected from PeerTube player.' + }) + + this.downloadedBytesHTTPCounter = this.meter.createCounter('peertube_playback_http_downloaded_bytes', { + description: 'Downloaded bytes with HTTP by PeerTube player.' + }) + this.downloadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_downloaded_bytes', { + description: 'Downloaded bytes with P2P by PeerTube player.' + }) + + this.uploadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_uploaded_bytes', { + description: 'Uploaded bytes with P2P by PeerTube player.' + }) + } + + observe (video: MVideoImmutable, metrics: PlaybackMetricCreate) { + const attributes = { + videoOrigin: video.remote + ? 'remote' + : 'local', + + playerMode: metrics.playerMode, + + resolution: metrics.resolution + '', + fps: metrics.fps + '' + } + + this.errorsCounter.add(metrics.errors, attributes) + this.resolutionChangesCounter.add(metrics.resolutionChanges, attributes) + + this.downloadedBytesHTTPCounter.add(metrics.downloadedBytesHTTP, attributes) + this.downloadedBytesP2PCounter.add(metrics.downloadedBytesP2P, attributes) + + this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes) + } +} diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts index ffe493670..ba33c9505 100644 --- a/server/lib/opentelemetry/metrics.ts +++ b/server/lib/opentelemetry/metrics.ts @@ -4,10 +4,13 @@ import { PrometheusExporter } from '@opentelemetry/exporter-prometheus' import { MeterProvider } from '@opentelemetry/sdk-metrics-base' import { logger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' +import { MVideoImmutable } from '@server/types/models' +import { PlaybackMetricCreate } from '@shared/models' import { JobQueueObserversBuilder, LivesObserversBuilder, NodeJSObserversBuilder, + PlaybackMetrics, StatsObserversBuilder, ViewersObserversBuilder } from './metric-helpers' @@ -20,6 +23,8 @@ class OpenTelemetryMetrics { private onRequestDuration: (req: Request, res: Response) => void + private playbackMetrics: PlaybackMetrics + private constructor () {} init (app: Application) { @@ -41,7 +46,11 @@ class OpenTelemetryMetrics { logger.info('Registering Open Telemetry metrics') - const provider = new MeterProvider() + const provider = new MeterProvider({ + views: [ + ...NodeJSObserversBuilder.getViews() + ] + }) provider.addMetricReader(new PrometheusExporter({ port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT })) @@ -51,7 +60,10 @@ class OpenTelemetryMetrics { this.buildRequestObserver() - const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter, provider) + this.playbackMetrics = new PlaybackMetrics(this.meter) + this.playbackMetrics.buildCounters() + + const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter) nodeJSObserversBuilder.buildObservers() const jobQueueObserversBuilder = new JobQueueObserversBuilder(this.meter) @@ -67,6 +79,10 @@ class OpenTelemetryMetrics { viewersObserversBuilder.buildObservers() } + observePlaybackMetric (video: MVideoImmutable, metrics: PlaybackMetricCreate) { + this.playbackMetrics.observe(video, metrics) + } + private buildRequestObserver () { const requestDuration = this.meter.createHistogram('http_request_duration_ms', { unit: 'milliseconds', diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index b0ad04819..ffadb3b49 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -10,6 +10,7 @@ export * from './express' export * from './feeds' export * from './follows' export * from './jobs' +export * from './metrics' export * from './logs' export * from './oembed' export * from './pagination' diff --git a/server/middlewares/validators/metrics.ts b/server/middlewares/validators/metrics.ts new file mode 100644 index 000000000..b1dbec603 --- /dev/null +++ b/server/middlewares/validators/metrics.ts @@ -0,0 +1,56 @@ +import express from 'express' +import { body } from 'express-validator' +import { isValidPlayerMode } from '@server/helpers/custom-validators/metrics' +import { isIdOrUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc' +import { CONFIG } from '@server/initializers/config' +import { HttpStatusCode, PlaybackMetricCreate } from '@shared/models' +import { logger } from '../../helpers/logger' +import { areValidationErrors, doesVideoExist } from './shared' + +const addPlaybackMetricValidator = [ + body('resolution') + .isInt({ min: 0 }).withMessage('Invalid resolution'), + body('fps') + .optional() + .isInt({ min: 0 }).withMessage('Invalid fps'), + body('playerMode') + .custom(isValidPlayerMode).withMessage('Invalid playerMode'), + + body('resolutionChanges') + .isInt({ min: 0 }).withMessage('Invalid resolutionChanges'), + + body('errors') + .isInt({ min: 0 }).withMessage('Invalid errors'), + + body('downloadedBytesP2P') + .isInt({ min: 0 }).withMessage('Invalid downloadedBytesP2P'), + body('downloadedBytesHTTP') + .isInt({ min: 0 }).withMessage('Invalid downloadedBytesHTTP'), + + body('uploadedBytesP2P') + .isInt({ min: 0 }).withMessage('Invalid uploadedBytesP2P'), + + body('videoId') + .customSanitizer(toCompleteUUID) + .optional() + .custom(isIdOrUUIDValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking addPlaybackMetricValidator parameters.', { parameters: req.query }) + + if (!CONFIG.OPEN_TELEMETRY.METRICS.ENABLED) return res.sendStatus(HttpStatusCode.NO_CONTENT_204) + + const body: PlaybackMetricCreate = req.body + + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(body.videoId, res, 'only-immutable-attributes')) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + addPlaybackMetricValidator +} diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index 149305f49..cd7a38459 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -10,6 +10,7 @@ import './follows' import './jobs' import './live' import './logs' +import './metrics' import './my-user' import './plugins' import './redundancy' diff --git a/server/tests/api/check-params/metrics.ts b/server/tests/api/check-params/metrics.ts new file mode 100644 index 000000000..2d4509406 --- /dev/null +++ b/server/tests/api/check-params/metrics.ts @@ -0,0 +1,183 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import { omit } from 'lodash' +import { HttpStatusCode, PlaybackMetricCreate, VideoResolution } from '@shared/models' +import { cleanupTests, createSingleServer, makePostBodyRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' + +describe('Test metrics API validators', function () { + let server: PeerTubeServer + let videoUUID: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1, { + open_telemetry: { + metrics: { + enabled: true + } + } + }) + + await setAccessTokensToServers([ server ]) + + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + videoUUID = uuid + }) + + describe('When adding playback metrics', function () { + const path = '/api/v1/metrics/playback' + let baseParams: PlaybackMetricCreate + + before(function () { + baseParams = { + playerMode: 'p2p-media-loader', + resolution: VideoResolution.H_1080P, + fps: 30, + resolutionChanges: 1, + errors: 2, + downloadedBytesP2P: 0, + downloadedBytesHTTP: 0, + uploadedBytesP2P: 0, + videoId: videoUUID + } + }) + + it('Should fail with an invalid resolution', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, resolution: 'toto' } + }) + }) + + it('Should fail with an invalid fps', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, fps: 'toto' } + }) + }) + + it('Should fail with a missing/invalid player mode', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, 'playerMode') + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, playerMode: 'toto' } + }) + }) + + it('Should fail with an missing/invalid resolution changes', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, 'resolutionChanges') + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, resolutionChanges: 'toto' } + }) + }) + + it('Should fail with a missing errors', async function () { + + }) + + it('Should fail with an missing/invalid errors', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, 'errors') + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, errors: 'toto' } + }) + }) + + it('Should fail with an missing/invalid downloadedBytesP2P', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, 'downloadedBytesP2P') + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, downloadedBytesP2P: 'toto' } + }) + }) + + it('Should fail with an missing/invalid downloadedBytesHTTP', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, 'downloadedBytesHTTP') + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, downloadedBytesHTTP: 'toto' } + }) + }) + + it('Should fail with an missing/invalid uploadedBytesP2P', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, 'uploadedBytesP2P') + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, uploadedBytesP2P: 'toto' } + }) + }) + + it('Should fail with a bad video id', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, videoId: 'toto' } + }) + }) + + it('Should fail with an unknown video', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, videoId: 42 }, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: baseParams, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts index 20909429f..3137a9eb6 100644 --- a/server/tests/api/server/open-telemetry.ts +++ b/server/tests/api/server/open-telemetry.ts @@ -2,14 +2,14 @@ import { expect } from 'chai' import { expectLogContain, expectLogDoesNotContain, MockHTTP } from '@server/tests/shared' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' +import { HttpStatusCode, VideoPrivacy, VideoResolution } from '@shared/models' import { cleanupTests, createSingleServer, makeRawRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' describe('Open Telemetry', function () { let server: PeerTubeServer describe('Metrics', function () { - const metricsUrl = 'http://localhost:9091/metrics' + const metricsUrl = 'http://localhost:9092/metrics' it('Should not enable open telemetry metrics', async function () { server = await createSingleServer(1) @@ -36,8 +36,33 @@ describe('Open Telemetry', function () { }) const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) - expect(res.text).to.contain('peertube_job_queue_total') + expect(res.text).to.contain('peertube_job_queue_total{') + }) + + it('Should have playback metrics', async function () { + await setAccessTokensToServers([ server ]) + + const video = await server.videos.quickUpload({ name: 'video' }) + + await server.metrics.addPlaybackMetric({ + metrics: { + playerMode: 'p2p-media-loader', + resolution: VideoResolution.H_1080P, + fps: 30, + resolutionChanges: 1, + errors: 2, + downloadedBytesP2P: 0, + downloadedBytesHTTP: 0, + uploadedBytesP2P: 5, + videoId: video.uuid + } + }) + const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) + expect(res.text).to.contain('peertube_playback_http_uploaded_bytes_total{') + }) + + after(async function () { await server.kill() }) }) -- cgit v1.2.3