From: Chocobozzz Date: Tue, 5 Jul 2022 13:43:21 +0000 (+0200) Subject: Introduce experimental telemetry X-Git-Tag: v4.3.0-rc.1~361 X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=630d0a1bf5897fff203cb07e426223f55dcc882d;p=github%2FChocobozzz%2FPeerTube.git Introduce experimental telemetry --- diff --git a/config/default.yaml b/config/default.yaml index 8e5d13355..8458ee016 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -177,6 +177,22 @@ log: log_tracker_unknown_infohash: true prettify_sql: false +# Highly experimental support of Open Telemetry +open_telemetry: + metrics: + enabled: false + + # Create a prometheus exporter server on this port so prometheus server can scrape PeerTube metrics + prometheus_exporter: + port: 9091 + + tracing: + enabled: false + + # Send traces to a Jaeger compatible endpoint + jaeger_exporter: + endpoint: '' + trending: videos: interval_days: 7 # Compute trending videos for the last x days diff --git a/config/production.yaml.example b/config/production.yaml.example index ef0358221..3da2421fe 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -175,6 +175,22 @@ log: log_tracker_unknown_infohash: true prettify_sql: false +# Highly experimental support of Open Telemetry +open_telemetry: + metrics: + enabled: false + + # Create a prometheus exporter server on this port so prometheus server can scrape PeerTube metrics + prometheus_exporter: + port: 9091 + + tracing: + enabled: false + + # Send traces to a Jaeger compatible endpoint + jaeger_exporter: + endpoint: '' + trending: videos: interval_days: 7 # Compute trending videos for the last x days diff --git a/package.json b/package.json index 6a5bcf75c..79bc7cf1f 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,22 @@ "@aws-sdk/node-http-handler": "^3.82.0", "@babel/parser": "7.17.8", "@node-oauth/oauth2-server": "^4.2.0", + "@opentelemetry/api": "^1.1.0", + "@opentelemetry/api-metrics": "^0.29.2", + "@opentelemetry/exporter-jaeger": "^1.3.1", + "@opentelemetry/exporter-prometheus": "~0.29.2", + "@opentelemetry/instrumentation": "^0.29.2", + "@opentelemetry/instrumentation-dns": "^0.29.0", + "@opentelemetry/instrumentation-express": "^0.30.0", + "@opentelemetry/instrumentation-fs": "^0.4.0", + "@opentelemetry/instrumentation-http": "^0.29.2", + "@opentelemetry/instrumentation-pg": "^0.30.0", + "@opentelemetry/instrumentation-redis-4": "^0.31.0", + "@opentelemetry/resources": "^1.3.1", + "@opentelemetry/sdk-metrics-base": "~0.29.2", + "@opentelemetry/sdk-trace-base": "^1.3.1", + "@opentelemetry/sdk-trace-node": "^1.3.1", + "@opentelemetry/semantic-conventions": "^1.3.1", "@peertube/feed": "^5.0.1", "@peertube/http-signature": "^1.6.0", "@uploadx/core": "^5.1.2", diff --git a/server.ts b/server.ts index 559327f16..73b7441f9 100644 --- a/server.ts +++ b/server.ts @@ -1,4 +1,7 @@ // ----------- Node modules ----------- +import { registerOpentelemetryTracing } from './server/lib/opentelemetry/tracing' +registerOpentelemetryTracing() + import express from 'express' import morgan, { token } from 'morgan' import cors from 'cors' @@ -47,6 +50,12 @@ checkConfig() // Trust our proxy (IP forwarding...) app.set('trust proxy', CONFIG.TRUST_PROXY) +app.use((_req, res, next) => { + res.locals.requestStart = Date.now() + + return next() +}) + // Security middleware import { baseCSP } from './server/middlewares/csp' @@ -126,6 +135,7 @@ import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache import { ServerConfigManager } from '@server/lib/server-config-manager' import { VideoViewsManager } from '@server/lib/views/video-views-manager' import { isTestInstance } from './server/helpers/core-utils' +import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics' // ----------- Command line ----------- @@ -194,6 +204,10 @@ app.use(cookieParser()) // W3C DNT Tracking Status app.use(advertiseDoNotTrack) +// ----------- Open Telemetry ----------- + +OpenTelemetryMetrics.Instance.init(app) + // ----------- Views, routes and static files ----------- // API @@ -297,6 +311,7 @@ async function startApplication () { RemoveDanglingResumableUploadsScheduler.Instance.enable() VideoViewsBufferScheduler.Instance.enable() GeoIPUpdateScheduler.Instance.enable() + OpenTelemetryMetrics.Instance.registerMetrics() Redis.Instance.init() PeerTubeSocket.Instance.init(server) diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 4fbaf8a73..9625c1b33 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -1,54 +1,18 @@ -// Thanks http://tostring.it/2014/06/23/advanced-logging-with-nodejs/ import { stat } from 'fs-extra' import { omit } from 'lodash' import { join } from 'path' import { format as sqlFormat } from 'sql-formatter' import { createLogger, format, transports } from 'winston' import { FileTransportOptions } from 'winston/lib/winston/transports' +import { context } from '@opentelemetry/api' +import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils' import { CONFIG } from '../initializers/config' import { LOG_FILENAME } from '../initializers/constants' const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT -function getLoggerReplacer () { - const seen = new WeakSet() - - // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples - return (key: string, value: any) => { - if (key === 'cert') return 'Replaced by the logger to avoid large log message' - - if (typeof value === 'object' && value !== null) { - if (seen.has(value)) return - - seen.add(value) - } - - if (value instanceof Set) { - return Array.from(value) - } - - if (value instanceof Map) { - return Array.from(value.entries()) - } - - if (value instanceof Error) { - const error = {} - - Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] }) - - return error - } - - return value - } -} - const consoleLoggerFormat = format.printf(info => { - const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ] - - const obj = omit(info, ...toOmit) - - let additionalInfos = JSON.stringify(obj, getLoggerReplacer(), 2) + let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2) if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' else additionalInfos = ' ' + additionalInfos @@ -68,7 +32,7 @@ const consoleLoggerFormat = format.printf(info => { }) const jsonLoggerFormat = format.printf(info => { - return JSON.stringify(info, getLoggerReplacer()) + return JSON.stringify(info, removeCyclicValues()) }) const timestampFormatter = format.timestamp({ @@ -94,11 +58,14 @@ if (CONFIG.LOG.ROTATION.ENABLED) { fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES } -const logger = buildLogger() - function buildLogger (labelSuffix?: string) { return createLogger({ level: CONFIG.LOG.LEVEL, + defaultMeta: { + get traceId () { return getSpanContext(context.active())?.traceId }, + get spanId () { return getSpanContext(context.active())?.spanId }, + get traceFlags () { return getSpanContext(context.active())?.traceFlags } + }, format: format.combine( labelFormatter(labelSuffix), format.splat() @@ -118,6 +85,10 @@ function buildLogger (labelSuffix?: string) { }) } +const logger = buildLogger() + +// --------------------------------------------------------------------------- + function bunyanLogFactory (level: string) { return function (...params: any[]) { let meta = null @@ -141,12 +112,15 @@ const bunyanLogger = { level: () => { }, trace: bunyanLogFactory('debug'), debug: bunyanLogFactory('debug'), + verbose: bunyanLogFactory('debug'), info: bunyanLogFactory('info'), warn: bunyanLogFactory('warn'), error: bunyanLogFactory('error'), fatal: bunyanLogFactory('error') } +// --------------------------------------------------------------------------- + type LoggerTagsFn = (...tags: string[]) => { tags: string[] } function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn { return (...tags: string[]) => { @@ -154,6 +128,8 @@ function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn { } } +// --------------------------------------------------------------------------- + async function mtimeSortFilesDesc (files: string[], basePath: string) { const promises = [] const out: { file: string, mtime: number }[] = [] @@ -189,3 +165,44 @@ export { loggerTagsFactory, bunyanLogger } + +// --------------------------------------------------------------------------- + +function removeCyclicValues () { + const seen = new WeakSet() + + // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples + return (key: string, value: any) => { + if (key === 'cert') return 'Replaced by the logger to avoid large log message' + + if (typeof value === 'object' && value !== null) { + if (seen.has(value)) return + + seen.add(value) + } + + if (value instanceof Set) { + return Array.from(value) + } + + if (value instanceof Map) { + return Array.from(value.entries()) + } + + if (value instanceof Error) { + const error = {} + + Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] }) + + return error + } + + return value + } +} + +function getAdditionalInfo (info: any) { + const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ] + + return omit(info, ...toOmit) +} diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 754585981..0943ffe2d 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -167,6 +167,22 @@ const CONFIG = { LOG_TRACKER_UNKNOWN_INFOHASH: config.get('log.log_tracker_unknown_infohash'), PRETTIFY_SQL: config.get('log.prettify_sql') }, + OPEN_TELEMETRY: { + METRICS: { + ENABLED: config.get('open_telemetry.metrics.enabled'), + + PROMETHEUS_EXPORTER: { + PORT: config.get('open_telemetry.metrics.prometheus_exporter.port') + } + }, + TRACING: { + ENABLED: config.get('open_telemetry.tracing.enabled'), + + JAEGER_EXPORTER: { + ENDPOINT: config.get('open_telemetry.tracing.jaeger_exporter.endpoint') + } + } + }, TRENDING: { VIDEOS: { INTERVAL_DAYS: config.get('trending.videos.interval_days'), diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index c6989c38b..e3683269c 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -736,7 +736,8 @@ const MEMOIZE_TTL = { INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours VIDEO_DURATION: 1000 * 10, // 10 seconds LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute - LIVE_CHECK_SOCKET_HEALTH: 1000 * 60 // 1 minute + LIVE_CHECK_SOCKET_HEALTH: 1000 * 60, // 1 minute + GET_STATS_FOR_OPEN_TELEMETRY_METRICS: 1000 * 60 // 1 minute } const MEMOIZE_LENGTH = { diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts index e6cec1ba7..ba2967ce9 100644 --- a/server/lib/activitypub/activity.ts +++ b/server/lib/activitypub/activity.ts @@ -1,3 +1,5 @@ +import { ActivityType } from "@shared/models" + function getAPId (object: string | { id: string }) { if (typeof object === 'string') return object @@ -13,8 +15,26 @@ function getDurationFromActivityStream (duration: string) { return parseInt(duration.replace(/[^\d]+/, '')) } +function buildAvailableActivities (): ActivityType[] { + return [ + 'Create', + 'Update', + 'Delete', + 'Follow', + 'Accept', + 'Announce', + 'Undo', + 'Like', + 'Reject', + 'View', + 'Dislike', + 'Flag' + ] +} + export { getAPId, getActivityStreamDuration, + buildAvailableActivities, getDurationFromActivityStream } diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index ce24763f1..e55d2e7c2 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -285,6 +285,12 @@ class JobQueue { return total } + async getStats () { + const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() })) + + return Promise.all(promises) + } + async removeOldJobs () { for (const key of Object.keys(this.queues)) { const queue = this.queues[key] diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts new file mode 100644 index 000000000..cabb27326 --- /dev/null +++ b/server/lib/opentelemetry/metric-helpers/index.ts @@ -0,0 +1 @@ +export * from './stats-observers-builder' diff --git a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts new file mode 100644 index 000000000..90b58f33d --- /dev/null +++ b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts @@ -0,0 +1,186 @@ +import memoizee from 'memoizee' +import { Meter } from '@opentelemetry/api-metrics' +import { MEMOIZE_TTL } from '@server/initializers/constants' +import { buildAvailableActivities } from '@server/lib/activitypub/activity' +import { StatsManager } from '@server/lib/stat-manager' + +export class StatsObserverBuilder { + + private readonly getInstanceStats = memoizee(() => { + return StatsManager.Instance.getStats() + }, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS }) + + constructor (private readonly meter: Meter) { + + } + + buildObservers () { + this.buildUserStatsObserver() + this.buildVideoStatsObserver() + this.buildCommentStatsObserver() + this.buildPlaylistStatsObserver() + this.buildChannelStatsObserver() + this.buildInstanceFollowsStatsObserver() + this.buildRedundancyStatsObserver() + this.buildActivityPubStatsObserver() + } + + private buildUserStatsObserver () { + this.meter.createObservableGauge('peertube_users_total', { + description: 'Total users on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalUsers) + }) + + this.meter.createObservableGauge('peertube_active_users_total', { + description: 'Total active users on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' }) + observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' }) + observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' }) + }) + } + + private buildChannelStatsObserver () { + this.meter.createObservableGauge('peertube_channels_total', { + description: 'Total channels on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' }) + }) + + this.meter.createObservableGauge('peertube_active_channels_total', { + description: 'Total active channels on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' }) + observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' }) + observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' }) + }) + } + + private buildVideoStatsObserver () { + this.meter.createObservableGauge('peertube_videos_total', { + description: 'Total videos on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' }) + observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' }) + }) + + this.meter.createObservableGauge('peertube_video_views_total', { + description: 'Total video views made on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' }) + }) + + this.meter.createObservableGauge('peertube_video_bytes_total', { + description: 'Total bytes of videos' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' }) + }) + } + + private buildCommentStatsObserver () { + this.meter.createObservableGauge('peertube_comments_total', { + description: 'Total comments on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' }) + }) + } + + private buildPlaylistStatsObserver () { + this.meter.createObservableGauge('peertube_playlists_total', { + description: 'Total playlists on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' }) + }) + } + + private buildInstanceFollowsStatsObserver () { + this.meter.createObservableGauge('peertube_instance_followers_total', { + description: 'Total followers of the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalInstanceFollowers) + }) + + this.meter.createObservableGauge('peertube_instance_following_total', { + description: 'Total following of the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalInstanceFollowing) + }) + } + + private buildRedundancyStatsObserver () { + this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', { + description: 'Total redundancy used of the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + for (const r of stats.videosRedundancy) { + observableResult.observe(r.totalUsed, { strategy: r.strategy }) + } + }) + + this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', { + description: 'Total redundancy available of the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + for (const r of stats.videosRedundancy) { + observableResult.observe(r.totalSize, { strategy: r.strategy }) + } + }) + } + + private buildActivityPubStatsObserver () { + const availableActivities = buildAvailableActivities() + + this.meter.createObservableGauge('peertube_ap_inbox_success_total', { + description: 'Total inbox messages processed with success' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + for (const type of availableActivities) { + observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type }) + } + }) + + this.meter.createObservableGauge('peertube_ap_inbox_error_total', { + description: 'Total inbox messages processed with error' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + for (const type of availableActivities) { + observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type }) + } + }) + + this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', { + description: 'Total inbox messages waiting for being processed' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalActivityPubMessagesWaiting) + }) + } +} diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts new file mode 100644 index 000000000..ca0aae8e7 --- /dev/null +++ b/server/lib/opentelemetry/metrics.ts @@ -0,0 +1,111 @@ +import { Application, Request, Response } from 'express' +import { Meter, metrics } from '@opentelemetry/api-metrics' +import { PrometheusExporter } from '@opentelemetry/exporter-prometheus' +import { MeterProvider } from '@opentelemetry/sdk-metrics-base' +import { logger } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { JobQueue } from '../job-queue' +import { StatsObserverBuilder } from './metric-helpers' + +class OpenTelemetryMetrics { + + private static instance: OpenTelemetryMetrics + + private meter: Meter + + private onRequestDuration: (req: Request, res: Response) => void + + private constructor () {} + + init (app: Application) { + if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return + + app.use((req, res, next) => { + res.once('finish', () => { + if (!this.onRequestDuration) return + + this.onRequestDuration(req as Request, res as Response) + }) + + next() + }) + } + + registerMetrics () { + if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return + + logger.info('Registering Open Telemetry metrics') + + const provider = new MeterProvider() + + provider.addMetricReader(new PrometheusExporter({ port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT })) + + metrics.setGlobalMeterProvider(provider) + + this.meter = metrics.getMeter('default') + + this.buildMemoryObserver() + this.buildRequestObserver() + this.buildJobQueueObserver() + + const statsObserverBuilder = new StatsObserverBuilder(this.meter) + statsObserverBuilder.buildObservers() + } + + private buildMemoryObserver () { + this.meter.createObservableGauge('nodejs_memory_usage_bytes', { + description: 'Memory' + }).addCallback(observableResult => { + const current = process.memoryUsage() + + observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' }) + observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' }) + observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' }) + observableResult.observe(current.external, { memoryType: 'external' }) + observableResult.observe(current.rss, { memoryType: 'rss' }) + }) + } + + private buildJobQueueObserver () { + this.meter.createObservableGauge('peertube_job_queue_total', { + description: 'Total jobs in the PeerTube job queue' + }).addCallback(async observableResult => { + const stats = await JobQueue.Instance.getStats() + + for (const { jobType, counts } of stats) { + for (const state of Object.keys(counts)) { + observableResult.observe(counts[state], { jobType, state }) + } + } + }) + } + + private buildRequestObserver () { + const requestDuration = this.meter.createHistogram('http_request_duration_ms', { + unit: 'milliseconds', + description: 'Duration of HTTP requests in ms' + }) + + this.onRequestDuration = (req: Request, res: Response) => { + const duration = Date.now() - res.locals.requestStart + + requestDuration.record(duration, { + path: this.buildRequestPath(req.originalUrl), + method: req.method, + statusCode: res.statusCode + '' + }) + } + } + + private buildRequestPath (path: string) { + return path.split('?')[0] + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +export { + OpenTelemetryMetrics +} diff --git a/server/lib/opentelemetry/tracing.ts b/server/lib/opentelemetry/tracing.ts new file mode 100644 index 000000000..5358d04de --- /dev/null +++ b/server/lib/opentelemetry/tracing.ts @@ -0,0 +1,81 @@ +import { diag, DiagLogLevel, trace } from '@opentelemetry/api' +import { JaegerExporter } from '@opentelemetry/exporter-jaeger' +import { registerInstrumentations } from '@opentelemetry/instrumentation' +import { DnsInstrumentation } from '@opentelemetry/instrumentation-dns' +import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express' +import FsInstrumentation from '@opentelemetry/instrumentation-fs' +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' +import { PgInstrumentation } from '@opentelemetry/instrumentation-pg' +import { RedisInstrumentation } from '@opentelemetry/instrumentation-redis-4' +import { Resource } from '@opentelemetry/resources' +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions' +import { logger } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' + +function registerOpentelemetryTracing () { + if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) return + + logger.info('Registering Open Telemetry tracing') + + const customLogger = (level: string) => { + return (message: string, ...args: unknown[]) => { + let fullMessage = message + + for (const arg of args) { + if (typeof arg === 'string') fullMessage += arg + else break + } + + logger[level](fullMessage) + } + } + + diag.setLogger({ + error: customLogger('error'), + warn: customLogger('warn'), + info: customLogger('info'), + debug: customLogger('debug'), + verbose: customLogger('verbose') + }, DiagLogLevel.INFO) + + const tracerProvider = new NodeTracerProvider({ + resource: new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'peertube' + }) + }) + + registerInstrumentations({ + tracerProvider: tracerProvider, + instrumentations: [ + new PgInstrumentation({ + enhancedDatabaseReporting: true + }), + new DnsInstrumentation(), + new HttpInstrumentation(), + new ExpressInstrumentation(), + new RedisInstrumentation({ + dbStatementSerializer: function (cmdName, cmdArgs) { + return [ cmdName, ...cmdArgs ].join(' ') + } + }), + new FsInstrumentation() + ] + }) + + tracerProvider.addSpanProcessor( + new BatchSpanProcessor( + new JaegerExporter({ endpoint: CONFIG.OPEN_TELEMETRY.TRACING.JAEGER_EXPORTER.ENDPOINT }) + ) + ) + + tracerProvider.register() +} + +const tracer = trace.getTracer('peertube') + +export { + registerOpentelemetryTracing, + tracer +} diff --git a/server/models/video/video.ts b/server/models/video/video.ts index e5f8b5fa2..4f711b2fa 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -24,7 +24,6 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { buildNSFWFilter } from '@server/helpers/express-utils' import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' import { LiveManager } from '@server/lib/live/live-manager' import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' @@ -134,9 +133,9 @@ import { VideoJobInfoModel } from './video-job-info' import { VideoLiveModel } from './video-live' import { VideoPlaylistElementModel } from './video-playlist-element' import { VideoShareModel } from './video-share' +import { VideoSourceModel } from './video-source' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' import { VideoTagModel } from './video-tag' -import { VideoSourceModel } from './video-source' export enum ScopeNames { FOR_API = 'FOR_API', @@ -1370,11 +1369,7 @@ export class VideoModel extends Model>> { } static async getStats () { - const totalLocalVideos = await VideoModel.count({ - where: { - remote: false - } - }) + const serverActor = await getServerActor() let totalLocalVideoViews = await VideoModel.sum('views', { where: { @@ -1385,19 +1380,26 @@ export class VideoModel extends Model>> { // Sequelize could return null... if (!totalLocalVideoViews) totalLocalVideoViews = 0 - const serverActor = await getServerActor() - - const { total: totalVideos } = await VideoModel.listForApi({ + const baseOptions = { start: 0, count: 0, sort: '-publishedAt', - nsfw: buildNSFWFilter(), + nsfw: null, + isLocal: true, displayOnlyForFollower: { actorId: serverActor.id, orLocalVideos: true } + } + + const { total: totalLocalVideos } = await VideoModel.listForApi({ + ...baseOptions, + + isLocal: true }) + const { total: totalVideos } = await VideoModel.listForApi(baseOptions) + return { totalLocalVideos, totalLocalVideoViews, diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts index 45be107ce..78522c246 100644 --- a/server/tests/api/server/index.ts +++ b/server/tests/api/server/index.ts @@ -17,5 +17,6 @@ import './slow-follows' import './stats' import './tracker' import './no-client' +import './open-telemetry' import './plugins' import './proxy' diff --git a/server/tests/api/server/no-client.ts b/server/tests/api/server/no-client.ts index 913907788..193f6c987 100644 --- a/server/tests/api/server/no-client.ts +++ b/server/tests/api/server/no-client.ts @@ -1,7 +1,6 @@ -import 'mocha' import request from 'supertest' -import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands' import { HttpStatusCode } from '@shared/models' +import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands' describe('Start and stop server without web client routes', function () { let server: PeerTubeServer diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts new file mode 100644 index 000000000..20909429f --- /dev/null +++ b/server/tests/api/server/open-telemetry.ts @@ -0,0 +1,95 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { expectLogContain, expectLogDoesNotContain, MockHTTP } from '@server/tests/shared' +import { HttpStatusCode, VideoPrivacy } from '@shared/models' +import { cleanupTests, createSingleServer, makeRawRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' + +describe('Open Telemetry', function () { + let server: PeerTubeServer + + describe('Metrics', function () { + const metricsUrl = 'http://localhost:9091/metrics' + + it('Should not enable open telemetry metrics', async function () { + server = await createSingleServer(1) + + let hasError = false + try { + await makeRawRequest(metricsUrl, HttpStatusCode.NOT_FOUND_404) + } catch (err) { + hasError = err.message.includes('ECONNREFUSED') + } + + expect(hasError).to.be.true + + await server.kill() + }) + + it('Should enable open telemetry metrics', async function () { + server = await createSingleServer(1, { + open_telemetry: { + metrics: { + enabled: true + } + } + }) + + const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) + expect(res.text).to.contain('peertube_job_queue_total') + + await server.kill() + }) + }) + + describe('Tracing', function () { + let mockHTTP: MockHTTP + let mockPort: number + + before(async function () { + mockHTTP = new MockHTTP() + mockPort = await mockHTTP.initialize() + }) + + it('Should enable open telemetry tracing', async function () { + server = await createSingleServer(1) + + await expectLogDoesNotContain(server, 'Registering Open Telemetry tracing') + + await server.kill() + }) + + it('Should enable open telemetry metrics', async function () { + server = await createSingleServer(1, { + open_telemetry: { + tracing: { + enabled: true, + jaeger_exporter: { + endpoint: 'http://localhost:' + mockPort + } + } + } + }) + + await expectLogContain(server, 'Registering Open Telemetry tracing') + }) + + it('Should upload a video and correctly works', async function () { + await setAccessTokensToServers([ server ]) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC }) + + const video = await server.videos.get({ id: uuid }) + + expect(video.name).to.equal('video') + }) + + after(async function () { + await mockHTTP.terminate() + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index 33b917f31..55ebc6c3e 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -29,6 +29,12 @@ async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { expect(content.toString()).to.not.contain(str) } +async function expectLogContain (server: PeerTubeServer, str: string) { + const content = await server.servers.getLogContent() + + expect(content.toString()).to.contain(str) +} + async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { const res = await makeGetRequest({ url, @@ -99,5 +105,6 @@ export { expectNotStartWith, checkBadStartPagination, checkBadCountPagination, - checkBadSortPagination + checkBadSortPagination, + expectLogContain } diff --git a/server/tests/shared/mock-servers/index.ts b/server/tests/shared/mock-servers/index.ts index abf4a8203..1fa983116 100644 --- a/server/tests/shared/mock-servers/index.ts +++ b/server/tests/shared/mock-servers/index.ts @@ -1,5 +1,6 @@ export * from './mock-429' export * from './mock-email' +export * from './mock-http' export * from './mock-instances-index' export * from './mock-joinpeertube-versions' export * from './mock-object-storage' diff --git a/server/tests/shared/mock-servers/mock-http.ts b/server/tests/shared/mock-servers/mock-http.ts new file mode 100644 index 000000000..b7a019e07 --- /dev/null +++ b/server/tests/shared/mock-servers/mock-http.ts @@ -0,0 +1,23 @@ +import express from 'express' +import { Server } from 'http' +import { getPort, randomListen, terminateServer } from './shared' + +export class MockHTTP { + private server: Server + + async initialize () { + const app = express() + + app.get('/*', (req: express.Request, res: express.Response, next: express.NextFunction) => { + return res.sendStatus(200) + }) + + this.server = await randomListen(app) + + return getPort(this.server) + } + + terminate () { + return terminateServer(this.server) + } +} diff --git a/server/types/express.d.ts b/server/types/express.d.ts index 27e532c31..8f8c65102 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts @@ -103,6 +103,8 @@ declare module 'express' { }) => void locals: { + requestStart: number + apicache: { content: string | Buffer write: Writable['write'] diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts index b1dcf2065..82f5a737f 100644 --- a/shared/models/server/server-stats.model.ts +++ b/shared/models/server/server-stats.model.ts @@ -1,5 +1,10 @@ +import { ActivityType } from '../activitypub' import { VideoRedundancyStrategyWithManual } from '../redundancy' -export interface ServerStats { + +type ActivityPubMessagesSuccess = Record<`totalActivityPub${ActivityType}MessagesSuccesses`, number> +type ActivityPubMessagesErrors = Record<`totalActivityPub${ActivityType}MessagesErrors`, number> + +export interface ServerStats extends ActivityPubMessagesSuccess, ActivityPubMessagesErrors { totalUsers: number totalDailyActiveUsers: number totalWeeklyActiveUsers: number @@ -29,32 +34,6 @@ export interface ServerStats { totalActivityPubMessagesSuccesses: number totalActivityPubMessagesErrors: number - totalActivityPubCreateMessagesSuccesses: number - totalActivityPubUpdateMessagesSuccesses: number - totalActivityPubDeleteMessagesSuccesses: number - totalActivityPubFollowMessagesSuccesses: number - totalActivityPubAcceptMessagesSuccesses: number - totalActivityPubRejectMessagesSuccesses: number - totalActivityPubAnnounceMessagesSuccesses: number - totalActivityPubUndoMessagesSuccesses: number - totalActivityPubLikeMessagesSuccesses: number - totalActivityPubDislikeMessagesSuccesses: number - totalActivityPubFlagMessagesSuccesses: number - totalActivityPubViewMessagesSuccesses: number - - totalActivityPubCreateMessagesErrors: number - totalActivityPubUpdateMessagesErrors: number - totalActivityPubDeleteMessagesErrors: number - totalActivityPubFollowMessagesErrors: number - totalActivityPubAcceptMessagesErrors: number - totalActivityPubRejectMessagesErrors: number - totalActivityPubAnnounceMessagesErrors: number - totalActivityPubUndoMessagesErrors: number - totalActivityPubLikeMessagesErrors: number - totalActivityPubDislikeMessagesErrors: number - totalActivityPubFlagMessagesErrors: number - totalActivityPubViewMessagesErrors: number - activityPubMessagesProcessedPerSecond: number totalActivityPubMessagesWaiting: number } diff --git a/yarn.lock b/yarn.lock index 4d9b5b4bd..530482b22 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1606,6 +1606,173 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@opentelemetry/api-metrics@0.29.2", "@opentelemetry/api-metrics@^0.29.2": + version "0.29.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/api-metrics/-/api-metrics-0.29.2.tgz#daa823e0965754222b49a6ae6133df8b39ff8fd2" + integrity sha512-yRdF5beqKuEdsPNoO7ijWCQ9HcyN0Tlgicf8RS6gzGOI54d6Hj7yKquJ6+X9XV+CSRbRWJYb+lOsXyso7uyX2g== + dependencies: + "@opentelemetry/api" "^1.0.0" + +"@opentelemetry/api@^1.0.0", "@opentelemetry/api@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.1.0.tgz#563539048255bbe1a5f4f586a4a10a1bb737f44a" + integrity sha512-hf+3bwuBwtXsugA2ULBc95qxrOqP2pOekLz34BJhcAKawt94vfeNyUKpYc0lZQ/3sCP6LqRa7UAdHA7i5UODzQ== + +"@opentelemetry/context-async-hooks@1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-1.3.1.tgz#6b5288b535178fb8e3053c00c30eb38d3fdff60b" + integrity sha512-NKUY3SGiEEIOD3EpB8erpEF4K1iyXkWald1vJMaa973+EPTASNSXvzf8hZa7nhnUVxYbxtTJqbSRsZFfbZpw4g== + +"@opentelemetry/core@1.3.1", "@opentelemetry/core@^1.0.0": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.3.1.tgz#6eef5c5efca9a4cd7daa0cd4c7ff28ca2317c8d7" + integrity sha512-k7lOC86N7WIyUZsUuSKZfFIrUtINtlauMGQsC1r7jNmcr0vVJGqK1ROBvt7WWMxLbpMnt1q2pXJO8tKu0b9auA== + dependencies: + "@opentelemetry/semantic-conventions" "1.3.1" + +"@opentelemetry/exporter-jaeger@^1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-jaeger/-/exporter-jaeger-1.3.1.tgz#00070e54abea51d5001acfaeb9fcfd537bf0d1d5" + integrity sha512-uJ9811zn5TTdazyTNc4xmcDnKC8H63VRGp23ujGTxBOCFUnFzfI/kUGUJ8/O7Xok9Ulop7wuuBW3onL1WedfjA== + dependencies: + "@opentelemetry/core" "1.3.1" + "@opentelemetry/sdk-trace-base" "1.3.1" + "@opentelemetry/semantic-conventions" "1.3.1" + jaeger-client "^3.15.0" + +"@opentelemetry/exporter-prometheus@~0.29.2": + version "0.29.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.29.2.tgz#70ca7fb37655ca57a580387607d5465b47e27ac3" + integrity sha512-E5sRfUM4rzbvjxdpL1H6YRtjr8wY8+/2R4NjfxPEwrENLeeQk87V1E+YFLqAS7TfFLW7Zr4lmmamunwn5THvQA== + dependencies: + "@opentelemetry/api-metrics" "0.29.2" + "@opentelemetry/core" "1.3.1" + "@opentelemetry/sdk-metrics-base" "0.29.2" + +"@opentelemetry/instrumentation-dns@^0.29.0": + version "0.29.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-dns/-/instrumentation-dns-0.29.0.tgz#efd21eb9d8938de97e225b52f8a432d6072b4096" + integrity sha512-3WTC4m6JKviaABiR3a+56WUMvrUp9WW9EYC0+LRpqm7RK/1a5bYq2Cozc2SlFYX9ZfWKMqGS39/fU24mKQ5toA== + dependencies: + "@opentelemetry/instrumentation" "^0.29.2" + "@opentelemetry/semantic-conventions" "^1.0.0" + semver "^7.3.2" + +"@opentelemetry/instrumentation-express@^0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-express/-/instrumentation-express-0.30.0.tgz#4f5f45ce47c8f1ac75741284d3e59c861ced4269" + integrity sha512-OsCfM+ThAXh3wzsyHgXyA5HUoLMdLd6Asix2Jx8yxniruU/Gq8y4Cz7aLy/vXNckXHWO3fwwL5gb7K3dykTnAQ== + dependencies: + "@opentelemetry/core" "^1.0.0" + "@opentelemetry/instrumentation" "^0.29.2" + "@opentelemetry/semantic-conventions" "^1.0.0" + "@types/express" "4.17.13" + +"@opentelemetry/instrumentation-fs@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.4.0.tgz#8d689f741c8cb706fd28306f5f70dab756d2f02c" + integrity sha512-AINjLsYifpBC5R0YrkS0aaWBcql5WYr+UGste6HLJlHiselA22hBE3zP2WX4y+24eIlvDFKybwlodiNrITV16Q== + dependencies: + "@opentelemetry/core" "^1.0.0" + "@opentelemetry/instrumentation" "^0.29.2" + "@opentelemetry/semantic-conventions" "^1.0.0" + +"@opentelemetry/instrumentation-http@^0.29.2": + version "0.29.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-http/-/instrumentation-http-0.29.2.tgz#c4750c33929d476c2a656f457c83d2144c5dd844" + integrity sha512-XIF9WCH03rp3vQjwXXVdTxlsXT2AG6LYfFKO8r2QC+w4F4KFuZa4J3VPYJ0L/a/6dWt34DA67eBh3l6Z1rMZrg== + dependencies: + "@opentelemetry/core" "1.3.1" + "@opentelemetry/instrumentation" "0.29.2" + "@opentelemetry/semantic-conventions" "1.3.1" + semver "^7.3.5" + +"@opentelemetry/instrumentation-pg@^0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.30.0.tgz#5941dd2817a846e7b5bc283112a9cf46728d1f44" + integrity sha512-RQ3cTTJnCBE/9GagjSpaM+yzxN25MvEwOxDFes3y8c1cqrMgqxukQLm3MbcqCQ8e1g/8d18+oyiEeBUjZJ5jnw== + dependencies: + "@opentelemetry/instrumentation" "^0.29.2" + "@opentelemetry/semantic-conventions" "^1.0.0" + "@types/pg" "8.6.1" + "@types/pg-pool" "2.0.3" + +"@opentelemetry/instrumentation-redis-4@^0.31.0": + version "0.31.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-redis-4/-/instrumentation-redis-4-0.31.0.tgz#3db1cfc177857fc208b73565df2d54065c69a67e" + integrity sha512-3DY6bkqKnVlPc2WWHelb6DnU78ryYLQFqv0lqnVsoSkr7b6hnmw1Bzuwo/5YmS4C3XuTAD4/6dZVrQJ23g8HNA== + dependencies: + "@opentelemetry/instrumentation" "^0.29.2" + "@opentelemetry/semantic-conventions" "^1.0.0" + +"@opentelemetry/instrumentation@0.29.2", "@opentelemetry/instrumentation@^0.29.2": + version "0.29.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.29.2.tgz#70e6d4e1a84508f5e9d8c7c426adcd7b0dba6c95" + integrity sha512-LXx5V0ONNATQFCE8C5uqnxWSm4rcXLssdLHdXjtGdxRmURqj/JO8jYefqXCD0LzsqEQ6yxOx2GZ0dgXvhBVdTw== + dependencies: + "@opentelemetry/api-metrics" "0.29.2" + require-in-the-middle "^5.0.3" + semver "^7.3.2" + shimmer "^1.2.1" + +"@opentelemetry/propagator-b3@1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-b3/-/propagator-b3-1.3.1.tgz#39208de42afef5635e74f4bedca5961d6ce25004" + integrity sha512-tEAtHsRr6l3glsmKaJiJ/7HDw/isPv+f8OBsWJqkSlfLicKes8T/1D7nEDC6jPACiEbD3f6oK1KQSpMijC9/UQ== + dependencies: + "@opentelemetry/core" "1.3.1" + +"@opentelemetry/propagator-jaeger@1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.3.1.tgz#ad02cf5e63f7adb6986418dac916e7b89c34df5b" + integrity sha512-H6swQcjZ8aMCS5caZaEBaadfn205IqLlB3ZyY+tCWDf5YPwJgPpjw3qgYgWulHVSEzK7VQTle/mZG7u9MAe6Pw== + dependencies: + "@opentelemetry/core" "1.3.1" + +"@opentelemetry/resources@1.3.1", "@opentelemetry/resources@^1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-1.3.1.tgz#9fd85ac4ffeefc35441404b384d5c1db8b243121" + integrity sha512-X8bl3X0YjlsHWy0Iv0KUETtZuRUznX4yr1iScKCtfy8AoRfZFc2xxWKMDJ0TrqYwSapgeg4YwpmRzUKmmnrbeA== + dependencies: + "@opentelemetry/core" "1.3.1" + "@opentelemetry/semantic-conventions" "1.3.1" + +"@opentelemetry/sdk-metrics-base@0.29.2", "@opentelemetry/sdk-metrics-base@~0.29.2": + version "0.29.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-metrics-base/-/sdk-metrics-base-0.29.2.tgz#bd515455f1d90e211458dcf957f0ae937772b155" + integrity sha512-7hhhZ/6YRRgAXOUTeCsbe6SIk3wZAdAHnEwGGp7aiVH5AOyioHyHInw4EHtowlD6dbLxUWURjh6k+Geht2zbxg== + dependencies: + "@opentelemetry/api-metrics" "0.29.2" + "@opentelemetry/core" "1.3.1" + "@opentelemetry/resources" "1.3.1" + lodash.merge "4.6.2" + +"@opentelemetry/sdk-trace-base@1.3.1", "@opentelemetry/sdk-trace-base@^1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.3.1.tgz#958083dbab928eefd17848959ac8810c787bec7f" + integrity sha512-Or95QZ+9QyvAiwqj+K68z8bDDuyWF50c37w17D10GV1dWzg4Ezcectsu/GB61QcBxm3Y4br0EN5F5TpIFfFliQ== + dependencies: + "@opentelemetry/core" "1.3.1" + "@opentelemetry/resources" "1.3.1" + "@opentelemetry/semantic-conventions" "1.3.1" + +"@opentelemetry/sdk-trace-node@^1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.3.1.tgz#ef6598aef93b24bfaae1ddc0321d3cf00d44c304" + integrity sha512-4sn/pYhaVaEI8WY0arivM77858IM5BjUKvymjJ+HmRNWBocCJKCCCY4P9cL8w8iCGGmst5yxecMyvM7OOFBnmg== + dependencies: + "@opentelemetry/context-async-hooks" "1.3.1" + "@opentelemetry/core" "1.3.1" + "@opentelemetry/propagator-b3" "1.3.1" + "@opentelemetry/propagator-jaeger" "1.3.1" + "@opentelemetry/sdk-trace-base" "1.3.1" + semver "^7.3.5" + +"@opentelemetry/semantic-conventions@1.3.1", "@opentelemetry/semantic-conventions@^1.0.0", "@opentelemetry/semantic-conventions@^1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.3.1.tgz#ba07b864a3c955f061aa30ea3ef7f4ae4449794a" + integrity sha512-wU5J8rUoo32oSef/rFpOT1HIjLjAv3qIDHkw1QIhODV3OpAVHi5oVzlouozg9obUmZKtbZ0qUe/m7FP0y0yBzA== + "@peertube/feed@^5.0.1": version "5.0.2" resolved "https://registry.yarnpkg.com/@peertube/feed/-/feed-5.0.2.tgz#d9ae7f38f1ccc75d353a5e24ad335a982bc4df74" @@ -1859,7 +2026,7 @@ "@types/qs" "*" "@types/range-parser" "*" -"@types/express@*": +"@types/express@*", "@types/express@4.17.13": version "4.17.13" resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.13.tgz#a76e2995728999bab51a33fabce1d705a3709034" integrity sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA== @@ -2061,6 +2228,31 @@ dependencies: "@types/node" "*" +"@types/pg-pool@2.0.3": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@types/pg-pool/-/pg-pool-2.0.3.tgz#3eb8df2933f617f219a53091ad4080c94ba1c959" + integrity sha512-fwK5WtG42Yb5RxAwxm3Cc2dJ39FlgcaNiXKvtTLAwtCn642X7dgel+w1+cLWwpSOFImR3YjsZtbkfjxbHtFAeg== + dependencies: + "@types/pg" "*" + +"@types/pg@*": + version "8.6.5" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.6.5.tgz#2dce9cb468a6a5e0f1296a59aea3ac75dd27b702" + integrity sha512-tOkGtAqRVkHa/PVZicq67zuujI4Oorfglsr2IbKofDwBSysnaqSx7W1mDqFqdkGE6Fbgh+PZAl0r/BWON/mozw== + dependencies: + "@types/node" "*" + pg-protocol "*" + pg-types "^2.2.0" + +"@types/pg@8.6.1": + version "8.6.1" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.6.1.tgz#099450b8dc977e8197a44f5229cedef95c8747f9" + integrity sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w== + dependencies: + "@types/node" "*" + pg-protocol "*" + pg-types "^2.2.0" + "@types/qs@*": version "6.9.7" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" @@ -2441,6 +2633,11 @@ ajv@^8.6.3: require-from-string "^2.0.2" uri-js "^4.2.2" +ansi-color@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/ansi-color/-/ansi-color-0.2.1.tgz#3e75c037475217544ed763a8db5709fa9ae5bf9a" + integrity sha512-bF6xLaZBLpOQzgYUtYEhJx090nPSZk1BQ/q2oyBK9aMMcJHzx9uXGCjI2Y+LebsN4Jwoykr0V9whbPiogdyHoQ== + ansi-colors@4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" @@ -2977,6 +3174,16 @@ bufferutil@^4.0.3: dependencies: node-gyp-build "^4.3.0" +bufrw@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/bufrw/-/bufrw-1.3.0.tgz#28d6cfdaf34300376836310f5c31d57eeb40c8fa" + integrity sha512-jzQnSbdJqhIltU9O5KUiTtljP9ccw2u5ix59McQy4pV2xGhVLhRZIndY8GIrgh5HjXa6+QJ9AQhOd2QWQizJFQ== + dependencies: + ansi-color "^0.2.1" + error "^7.0.0" + hexer "^1.5.0" + xtend "^4.0.0" + bull@^4.1.0: version "4.8.4" resolved "https://registry.yarnpkg.com/bull/-/bull-4.8.4.tgz#c538610492050d5160dbd9180704145f135a0aa9" @@ -4029,6 +4236,21 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" +error@7.0.2: + version "7.0.2" + resolved "https://registry.yarnpkg.com/error/-/error-7.0.2.tgz#a5f75fff4d9926126ddac0ea5dc38e689153cb02" + integrity sha512-UtVv4l5MhijsYUxPJo4390gzfZvAnTHreNnDjnTZaKIiZ/SemXxAhBkYSKtWa5RtBXbLP8tMgn/n0RUa/H7jXw== + dependencies: + string-template "~0.2.1" + xtend "~4.0.0" + +error@^7.0.0: + version "7.2.1" + resolved "https://registry.yarnpkg.com/error/-/error-7.2.1.tgz#eab21a4689b5f684fc83da84a0e390de82d94894" + integrity sha512-fo9HBvWnx3NGUKMvMwB/CBCMMrfEJgbDTVDEkPygA3Bdd3lM1OyCd+rbQ8BwnpF6GdVeOLDNmyL4N5Bg80ZvdA== + dependencies: + string-template "~0.2.1" + es-abstract@^1.19.0, es-abstract@^1.19.1, es-abstract@^1.19.2, es-abstract@^1.19.5: version "1.20.1" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.20.1.tgz#027292cd6ef44bd12b1913b828116f54787d1814" @@ -5044,6 +5266,16 @@ helmet@^5.0.1: resolved "https://registry.yarnpkg.com/helmet/-/helmet-5.1.0.tgz#e98a5d4bf89ab8119c856018a3bcc82addadcd47" integrity sha512-klsunXs8rgNSZoaUrNeuCiWUxyc+wzucnEnFejUg3/A+CaF589k9qepLZZ1Jehnzig7YbD4hEuscGXuBY3fq+g== +hexer@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/hexer/-/hexer-1.5.0.tgz#b86ce808598e8a9d1892c571f3cedd86fc9f0653" + integrity sha512-dyrPC8KzBzUJ19QTIo1gXNqIISRXQ0NwteW6OeQHRN4ZuZeHkdODfj0zHBdOlHbRY8GqbqK57C9oWSvQZizFsg== + dependencies: + ansi-color "^0.2.1" + minimist "^1.1.0" + process "^0.10.0" + xtend "^4.0.0" + hexoid@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/hexoid/-/hexoid-1.0.0.tgz#ad10c6573fb907de23d9ec63a711267d9dc9bc18" @@ -5563,6 +5795,17 @@ isstream@0.1.x: resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== +jaeger-client@^3.15.0: + version "3.19.0" + resolved "https://registry.yarnpkg.com/jaeger-client/-/jaeger-client-3.19.0.tgz#9b5bd818ebd24e818616ee0f5cffe1722a53ae6e" + integrity sha512-M0c7cKHmdyEUtjemnJyx/y9uX16XHocL46yQvyqDlPdvAcwPDbHrIbKjQdBqtiE4apQ/9dmr+ZLJYYPGnurgpw== + dependencies: + node-int64 "^0.4.0" + opentracing "^0.14.4" + thriftrw "^3.5.0" + uuid "^8.3.2" + xorshift "^1.1.1" + jimp@^0.16.0: version "0.16.1" resolved "https://registry.yarnpkg.com/jimp/-/jimp-0.16.1.tgz#192f851a30e5ca11112a3d0aa53137659a78ca7a" @@ -5900,7 +6143,7 @@ lodash.isarguments@^3.1.0: resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== -lodash.merge@^4.6.2: +lodash.merge@4.6.2, lodash.merge@^4.6.2: version "4.6.2" resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== @@ -5929,6 +6172,11 @@ logform@^2.3.2, logform@^2.4.0: safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" +long@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/long/-/long-2.4.0.tgz#9fa180bb1d9500cdc29c4156766a1995e1f4524f" + integrity sha512-ijUtjmO/n2A5PaosNG9ZGDsQ3vxJg7ZW8vsY8Kp0f2yIZWhSJvjmegV7t+9RPQKxKrvj8yKGehhS+po14hPLGQ== + loose-envify@^1.0.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -6320,6 +6568,11 @@ mocha@^10.0.0: yargs-parser "20.2.4" yargs-unparser "2.0.0" +module-details-from-path@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.3.tgz#114c949673e2a8a35e9d35788527aa37b679da2b" + integrity sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A== + moment-timezone@^0.5.34: version "0.5.34" resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.34.tgz#a75938f7476b88f155d3504a9343f7519d9a405c" @@ -6556,6 +6809,11 @@ node-gyp-build@^4.2.0, node-gyp-build@^4.2.2, node-gyp-build@^4.3.0: resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.4.0.tgz#42e99687ce87ddeaf3a10b99dc06abc11021f3f4" integrity sha512-amJnQCcgtRVw9SvoebO3BKGESClrfXGCUTX9hSn1OuGQTQBOZmVd0Z0OlecpuRksKvbsUqALE8jls/ErClAPuQ== +node-int64@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" + integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== + node-media-server@^2.1.4: version "2.3.12" resolved "https://registry.yarnpkg.com/node-media-server/-/node-media-server-2.3.12.tgz#5380a60a26545144fa1e7954f08ece6f255d28b4" @@ -6713,6 +6971,11 @@ open@7: is-docker "^2.0.0" is-wsl "^2.1.1" +opentracing@^0.14.4: + version "0.14.7" + resolved "https://registry.yarnpkg.com/opentracing/-/opentracing-0.14.7.tgz#25d472bd0296dc0b64d7b94cbc995219031428f5" + integrity sha512-vz9iS7MJ5+Bp1URw8Khvdyw1H/hGvzHWlKQ7eRrQojSCDL1/SrWfrY9QebLw97n2deyRtzHRC3MkQfVNUCo91Q== + optionator@^0.9.1: version "0.9.1" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" @@ -6998,12 +7261,12 @@ pg-pool@^3.5.1: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.5.1.tgz#f499ce76f9bf5097488b3b83b19861f28e4ed905" integrity sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ== -pg-protocol@^1.5.0: +pg-protocol@*, pg-protocol@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.5.0.tgz#b5dd452257314565e2d54ab3c132adc46565a6a0" integrity sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ== -pg-types@^2.1.0: +pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== @@ -7148,6 +7411,11 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +process@^0.10.0: + version "0.10.1" + resolved "https://registry.yarnpkg.com/process/-/process-0.10.1.tgz#842457cc51cfed72dc775afeeafb8c6034372725" + integrity sha512-dyIett8dgGIZ/TXKUzeYExt7WA6ldDzys9vTDU/cCA9L17Ypme+KzS+NjQCjpn9xsvi/shbMC+yP/BcFMBz0NA== + process@^0.11.10: version "0.11.10" resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" @@ -7618,6 +7886,15 @@ require-from-string@^2.0.2: resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== +require-in-the-middle@^5.0.3: + version "5.1.0" + resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-5.1.0.tgz#b768f800377b47526d026bbf5a7f727f16eb412f" + integrity sha512-M2rLKVupQfJ5lf9OvqFGIT+9iVLnTmjgbOmpil12hiSQNn5zJTKGPoIisETNjfK+09vP3rpm1zJajmErpr2sEQ== + dependencies: + debug "^4.1.1" + module-details-from-path "^1.0.3" + resolve "^1.12.0" + require-main-filename@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" @@ -7647,7 +7924,7 @@ resolve-tspaths@^0.6.0: commander "9.2.0" fast-glob "3.2.11" -resolve@^1.10.1, resolve@^1.15.1, resolve@^1.18.1, resolve@^1.20.0, resolve@^1.22.0: +resolve@^1.10.1, resolve@^1.12.0, resolve@^1.15.1, resolve@^1.18.1, resolve@^1.20.0, resolve@^1.22.0: version "1.22.1" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177" integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw== @@ -7910,6 +8187,11 @@ shell-quote@^1.7.3: resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.3.tgz#aa40edac170445b9a431e17bb62c0b881b9c4123" integrity sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw== +shimmer@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" + integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== + short-uuid@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/short-uuid/-/short-uuid-4.2.0.tgz#3706d9e7287ac589dc5ffe324d3e34817a07540b" @@ -8258,6 +8540,11 @@ string-argv@^0.1.1: resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.1.2.tgz#c5b7bc03fb2b11983ba3a72333dd0559e77e4738" integrity sha512-mBqPGEOMNJKXRo7z0keX0wlAhbBAjilUdPW13nN0PecVryZxdHIeM7TqbsSUA7VYuS00HGC6mojP7DlQzfa9ZA== +string-template@~0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/string-template/-/string-template-0.2.1.tgz#42932e598a352d01fc22ec3367d9d84eec6c9add" + integrity sha512-Yptehjogou2xm4UJbxJ4CxgZx12HBfeystp0y3x7s4Dj32ltVVG1Gg8YhKjHZkHicuKpZX/ffilA8505VbUbpw== + "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" @@ -8431,6 +8718,15 @@ thirty-two@^1.0.2: resolved "https://registry.yarnpkg.com/thirty-two/-/thirty-two-1.0.2.tgz#4ca2fffc02a51290d2744b9e3f557693ca6b627a" integrity sha512-OEI0IWCe+Dw46019YLl6V10Us5bi574EvlJEOcAkB29IzQ/mYD1A6RyNHLjZPiHCmuodxvgF6U+vZO1L15lxVA== +thriftrw@^3.5.0: + version "3.12.0" + resolved "https://registry.yarnpkg.com/thriftrw/-/thriftrw-3.12.0.tgz#30857847755e7f036b2e0a79d11c9f55075539d9" + integrity sha512-4YZvR4DPEI41n4Opwr4jmrLGG4hndxr7387kzRFIIzxHQjarPusH4lGXrugvgb7TtPrfZVTpZCVe44/xUxowEw== + dependencies: + bufrw "^1.3.0" + error "7.0.2" + long "^2.4.0" + through2@^0.6.3, through2@~0.6.1: version "0.6.5" resolved "https://registry.yarnpkg.com/through2/-/through2-0.6.5.tgz#41ab9c67b29d57209071410e1d7a7a968cd3ad48" @@ -9202,7 +9498,12 @@ xmlhttprequest-ssl@~2.0.0: resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67" integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A== -"xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@~4.0.1: +xorshift@^1.1.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/xorshift/-/xorshift-1.2.0.tgz#30a4cdd8e9f8d09d959ed2a88c42a09c660e8148" + integrity sha512-iYgNnGyeeJ4t6U11NpA/QiKy+PXn5Aa3Azg5qkwIFz1tBLllQrjjsk9yzD7IAK0naNU4JxdeDgqW9ov4u/hc4g== + +"xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@~4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==