Introduce experimental telemetry

17 files changed, 626 insertions, 57 deletions

diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 4fbaf8a73..9625c1b33 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -1,54 +1,18 @@
-// Thanks http://tostring.it/2014/06/23/advanced-logging-with-nodejs/
 import { stat } from 'fs-extra'
 import { omit } from 'lodash'
 import { join } from 'path'
 import { format as sqlFormat } from 'sql-formatter'
 import { createLogger, format, transports } from 'winston'
 import { FileTransportOptions } from 'winston/lib/winston/transports'
+import { context } from '@opentelemetry/api'
+import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils'
 import { CONFIG } from '../initializers/config'
 import { LOG_FILENAME } from '../initializers/constants'
 
 const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
 
-function getLoggerReplacer () {
-  const seen = new WeakSet()
-
-  // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples
-  return (key: string, value: any) => {
-    if (key === 'cert') return 'Replaced by the logger to avoid large log message'
-
-    if (typeof value === 'object' && value !== null) {
-      if (seen.has(value)) return
-
-      seen.add(value)
-    }
-
-    if (value instanceof Set) {
-      return Array.from(value)
-    }
-
-    if (value instanceof Map) {
-      return Array.from(value.entries())
-    }
-
-    if (value instanceof Error) {
-      const error = {}
-
-      Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] })
-
-      return error
-    }
-
-    return value
-  }
-}
-
 const consoleLoggerFormat = format.printf(info => {
-  const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ]
-
-  const obj = omit(info, ...toOmit)
-
-  let additionalInfos = JSON.stringify(obj, getLoggerReplacer(), 2)
+  let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2)
 
   if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
   else additionalInfos = ' ' + additionalInfos
@@ -68,7 +32,7 @@ const consoleLoggerFormat = format.printf(info => {
 })
 
 const jsonLoggerFormat = format.printf(info => {
-  return JSON.stringify(info, getLoggerReplacer())
+  return JSON.stringify(info, removeCyclicValues())
 })
 
 const timestampFormatter = format.timestamp({
@@ -94,11 +58,14 @@ if (CONFIG.LOG.ROTATION.ENABLED) {
   fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES
 }
 
-const logger = buildLogger()
-
 function buildLogger (labelSuffix?: string) {
   return createLogger({
     level: CONFIG.LOG.LEVEL,
+    defaultMeta: {
+      get traceId () { return getSpanContext(context.active())?.traceId },
+      get spanId () { return getSpanContext(context.active())?.spanId },
+      get traceFlags () { return getSpanContext(context.active())?.traceFlags }
+    },
     format: format.combine(
       labelFormatter(labelSuffix),
       format.splat()
@@ -118,6 +85,10 @@ function buildLogger (labelSuffix?: string) {
   })
 }
 
+const logger = buildLogger()
+
+// ---------------------------------------------------------------------------
+
 function bunyanLogFactory (level: string) {
   return function (...params: any[]) {
     let meta = null
@@ -141,12 +112,15 @@ const bunyanLogger = {
   level: () => { },
   trace: bunyanLogFactory('debug'),
   debug: bunyanLogFactory('debug'),
+  verbose: bunyanLogFactory('debug'),
   info: bunyanLogFactory('info'),
   warn: bunyanLogFactory('warn'),
   error: bunyanLogFactory('error'),
   fatal: bunyanLogFactory('error')
 }
 
+// ---------------------------------------------------------------------------
+
 type LoggerTagsFn = (...tags: string[]) => { tags: string[] }
 function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
   return (...tags: string[]) => {
@@ -154,6 +128,8 @@ function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
   }
 }
 
+// ---------------------------------------------------------------------------
+
 async function mtimeSortFilesDesc (files: string[], basePath: string) {
   const promises = []
   const out: { file: string, mtime: number }[] = []
@@ -189,3 +165,44 @@ export {
   loggerTagsFactory,
   bunyanLogger
 }
+
+// ---------------------------------------------------------------------------
+
+function removeCyclicValues () {
+  const seen = new WeakSet()
+
+  // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples
+  return (key: string, value: any) => {
+    if (key === 'cert') return 'Replaced by the logger to avoid large log message'
+
+    if (typeof value === 'object' && value !== null) {
+      if (seen.has(value)) return
+
+      seen.add(value)
+    }
+
+    if (value instanceof Set) {
+      return Array.from(value)
+    }
+
+    if (value instanceof Map) {
+      return Array.from(value.entries())
+    }
+
+    if (value instanceof Error) {
+      const error = {}
+
+      Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] })
+
+      return error
+    }
+
+    return value
+  }
+}
+
+function getAdditionalInfo (info: any) {
+  const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ]
+
+  return omit(info, ...toOmit)
+}
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 754585981..0943ffe2d 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -167,6 +167,22 @@ const CONFIG = {
     LOG_TRACKER_UNKNOWN_INFOHASH: config.get<boolean>('log.log_tracker_unknown_infohash'),
     PRETTIFY_SQL: config.get<boolean>('log.prettify_sql')
   },
+  OPEN_TELEMETRY: {
+    METRICS: {
+      ENABLED: config.get<boolean>('open_telemetry.metrics.enabled'),
+
+      PROMETHEUS_EXPORTER: {
+        PORT: config.get<number>('open_telemetry.metrics.prometheus_exporter.port')
+      }
+    },
+    TRACING: {
+      ENABLED: config.get<boolean>('open_telemetry.tracing.enabled'),
+
+      JAEGER_EXPORTER: {
+        ENDPOINT: config.get<string>('open_telemetry.tracing.jaeger_exporter.endpoint')
+      }
+    }
+  },
   TRENDING: {
     VIDEOS: {
       INTERVAL_DAYS: config.get<number>('trending.videos.interval_days'),
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index c6989c38b..e3683269c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -736,7 +736,8 @@ const MEMOIZE_TTL = {
   INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours
   VIDEO_DURATION: 1000 * 10, // 10 seconds
   LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute
-  LIVE_CHECK_SOCKET_HEALTH: 1000 * 60 // 1 minute
+  LIVE_CHECK_SOCKET_HEALTH: 1000 * 60, // 1 minute
+  GET_STATS_FOR_OPEN_TELEMETRY_METRICS: 1000 * 60 // 1 minute
 }
 
 const MEMOIZE_LENGTH = {
diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts
index e6cec1ba7..ba2967ce9 100644
--- a/server/lib/activitypub/activity.ts
+++ b/server/lib/activitypub/activity.ts
@@ -1,3 +1,5 @@
+import { ActivityType } from "@shared/models"
+
 function getAPId (object: string | { id: string }) {
   if (typeof object === 'string') return object
 
@@ -13,8 +15,26 @@ function getDurationFromActivityStream (duration: string) {
   return parseInt(duration.replace(/[^\d]+/, ''))
 }
 
+function buildAvailableActivities (): ActivityType[] {
+  return [
+    'Create',
+    'Update',
+    'Delete',
+    'Follow',
+    'Accept',
+    'Announce',
+    'Undo',
+    'Like',
+    'Reject',
+    'View',
+    'Dislike',
+    'Flag'
+  ]
+}
+
 export {
   getAPId,
   getActivityStreamDuration,
+  buildAvailableActivities,
   getDurationFromActivityStream
 }
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ce24763f1..e55d2e7c2 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -285,6 +285,12 @@ class JobQueue {
     return total
   }
 
+  async getStats () {
+    const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() }))
+
+    return Promise.all(promises)
+  }
+
   async removeOldJobs () {
     for (const key of Object.keys(this.queues)) {
       const queue = this.queues[key]
diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts
new file mode 100644
index 000000000..cabb27326
--- /dev/null
+++ b/server/lib/opentelemetry/metric-helpers/index.ts
@@ -0,0 +1 @@
+export * from './stats-observers-builder'
diff --git a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts
new file mode 100644
index 000000000..90b58f33d
--- /dev/null
+++ b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts
@@ -0,0 +1,186 @@
+import memoizee from 'memoizee'
+import { Meter } from '@opentelemetry/api-metrics'
+import { MEMOIZE_TTL } from '@server/initializers/constants'
+import { buildAvailableActivities } from '@server/lib/activitypub/activity'
+import { StatsManager } from '@server/lib/stat-manager'
+
+export class StatsObserverBuilder {
+
+  private readonly getInstanceStats = memoizee(() => {
+    return StatsManager.Instance.getStats()
+  }, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS })
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildObservers () {
+    this.buildUserStatsObserver()
+    this.buildVideoStatsObserver()
+    this.buildCommentStatsObserver()
+    this.buildPlaylistStatsObserver()
+    this.buildChannelStatsObserver()
+    this.buildInstanceFollowsStatsObserver()
+    this.buildRedundancyStatsObserver()
+    this.buildActivityPubStatsObserver()
+  }
+
+  private buildUserStatsObserver () {
+    this.meter.createObservableGauge('peertube_users_total', {
+      description: 'Total users on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalUsers)
+    })
+
+    this.meter.createObservableGauge('peertube_active_users_total', {
+      description: 'Total active users on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' })
+      observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' })
+      observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' })
+    })
+  }
+
+  private buildChannelStatsObserver () {
+    this.meter.createObservableGauge('peertube_channels_total', {
+      description: 'Total channels on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' })
+    })
+
+    this.meter.createObservableGauge('peertube_active_channels_total', {
+      description: 'Total active channels on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' })
+      observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' })
+      observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' })
+    })
+  }
+
+  private buildVideoStatsObserver () {
+    this.meter.createObservableGauge('peertube_videos_total', {
+      description: 'Total videos on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' })
+      observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' })
+    })
+
+    this.meter.createObservableGauge('peertube_video_views_total', {
+      description: 'Total video views made on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' })
+    })
+
+    this.meter.createObservableGauge('peertube_video_bytes_total', {
+      description: 'Total bytes of videos'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' })
+    })
+  }
+
+  private buildCommentStatsObserver () {
+    this.meter.createObservableGauge('peertube_comments_total', {
+      description: 'Total comments on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' })
+    })
+  }
+
+  private buildPlaylistStatsObserver () {
+    this.meter.createObservableGauge('peertube_playlists_total', {
+      description: 'Total playlists on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' })
+    })
+  }
+
+  private buildInstanceFollowsStatsObserver () {
+    this.meter.createObservableGauge('peertube_instance_followers_total', {
+      description: 'Total followers of the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalInstanceFollowers)
+    })
+
+    this.meter.createObservableGauge('peertube_instance_following_total', {
+      description: 'Total following of the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalInstanceFollowing)
+    })
+  }
+
+  private buildRedundancyStatsObserver () {
+    this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', {
+      description: 'Total redundancy used of the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      for (const r of stats.videosRedundancy) {
+        observableResult.observe(r.totalUsed, { strategy: r.strategy })
+      }
+    })
+
+    this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', {
+      description: 'Total redundancy available of the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      for (const r of stats.videosRedundancy) {
+        observableResult.observe(r.totalSize, { strategy: r.strategy })
+      }
+    })
+  }
+
+  private buildActivityPubStatsObserver () {
+    const availableActivities = buildAvailableActivities()
+
+    this.meter.createObservableGauge('peertube_ap_inbox_success_total', {
+      description: 'Total inbox messages processed with success'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      for (const type of availableActivities) {
+        observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type })
+      }
+    })
+
+    this.meter.createObservableGauge('peertube_ap_inbox_error_total', {
+      description: 'Total inbox messages processed with error'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      for (const type of availableActivities) {
+        observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type })
+      }
+    })
+
+    this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', {
+      description: 'Total inbox messages waiting for being processed'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalActivityPubMessagesWaiting)
+    })
+  }
+}
diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts
new file mode 100644
index 000000000..ca0aae8e7
--- /dev/null
+++ b/server/lib/opentelemetry/metrics.ts
@@ -0,0 +1,111 @@
+import { Application, Request, Response } from 'express'
+import { Meter, metrics } from '@opentelemetry/api-metrics'
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
+import { MeterProvider } from '@opentelemetry/sdk-metrics-base'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { JobQueue } from '../job-queue'
+import { StatsObserverBuilder } from './metric-helpers'
+
+class OpenTelemetryMetrics {
+
+  private static instance: OpenTelemetryMetrics
+
+  private meter: Meter
+
+  private onRequestDuration: (req: Request, res: Response) => void
+
+  private constructor () {}
+
+  init (app: Application) {
+    if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return
+
+    app.use((req, res, next) => {
+      res.once('finish', () => {
+        if (!this.onRequestDuration) return
+
+        this.onRequestDuration(req as Request, res as Response)
+      })
+
+      next()
+    })
+  }
+
+  registerMetrics () {
+    if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return
+
+    logger.info('Registering Open Telemetry metrics')
+
+    const provider = new MeterProvider()
+
+    provider.addMetricReader(new PrometheusExporter({ port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT }))
+
+    metrics.setGlobalMeterProvider(provider)
+
+    this.meter = metrics.getMeter('default')
+
+    this.buildMemoryObserver()
+    this.buildRequestObserver()
+    this.buildJobQueueObserver()
+
+    const statsObserverBuilder = new StatsObserverBuilder(this.meter)
+    statsObserverBuilder.buildObservers()
+  }
+
+  private buildMemoryObserver () {
+    this.meter.createObservableGauge('nodejs_memory_usage_bytes', {
+      description: 'Memory'
+    }).addCallback(observableResult => {
+      const current = process.memoryUsage()
+
+      observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' })
+      observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' })
+      observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' })
+      observableResult.observe(current.external, { memoryType: 'external' })
+      observableResult.observe(current.rss, { memoryType: 'rss' })
+    })
+  }
+
+  private buildJobQueueObserver () {
+    this.meter.createObservableGauge('peertube_job_queue_total', {
+      description: 'Total jobs in the PeerTube job queue'
+    }).addCallback(async observableResult => {
+      const stats = await JobQueue.Instance.getStats()
+
+      for (const { jobType, counts } of stats) {
+        for (const state of Object.keys(counts)) {
+          observableResult.observe(counts[state], { jobType, state })
+        }
+      }
+    })
+  }
+
+  private buildRequestObserver () {
+    const requestDuration = this.meter.createHistogram('http_request_duration_ms', {
+      unit: 'milliseconds',
+      description: 'Duration of HTTP requests in ms'
+    })
+
+    this.onRequestDuration = (req: Request, res: Response) => {
+      const duration = Date.now() - res.locals.requestStart
+
+      requestDuration.record(duration, {
+        path: this.buildRequestPath(req.originalUrl),
+        method: req.method,
+        statusCode: res.statusCode + ''
+      })
+    }
+  }
+
+  private buildRequestPath (path: string) {
+    return path.split('?')[0]
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}
+
+export {
+  OpenTelemetryMetrics
+}
diff --git a/server/lib/opentelemetry/tracing.ts b/server/lib/opentelemetry/tracing.ts
new file mode 100644
index 000000000..5358d04de
--- /dev/null
+++ b/server/lib/opentelemetry/tracing.ts
@@ -0,0 +1,81 @@
+import { diag, DiagLogLevel, trace } from '@opentelemetry/api'
+import { JaegerExporter } from '@opentelemetry/exporter-jaeger'
+import { registerInstrumentations } from '@opentelemetry/instrumentation'
+import { DnsInstrumentation } from '@opentelemetry/instrumentation-dns'
+import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express'
+import FsInstrumentation from '@opentelemetry/instrumentation-fs'
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
+import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'
+import { RedisInstrumentation } from '@opentelemetry/instrumentation-redis-4'
+import { Resource } from '@opentelemetry/resources'
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+
+function registerOpentelemetryTracing () {
+  if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) return
+
+  logger.info('Registering Open Telemetry tracing')
+
+  const customLogger = (level: string) => {
+    return (message: string, ...args: unknown[]) => {
+      let fullMessage = message
+
+      for (const arg of args) {
+        if (typeof arg === 'string') fullMessage += arg
+        else break
+      }
+
+      logger[level](fullMessage)
+    }
+  }
+
+  diag.setLogger({
+    error: customLogger('error'),
+    warn: customLogger('warn'),
+    info: customLogger('info'),
+    debug: customLogger('debug'),
+    verbose: customLogger('verbose')
+  }, DiagLogLevel.INFO)
+
+  const tracerProvider = new NodeTracerProvider({
+    resource: new Resource({
+      [SemanticResourceAttributes.SERVICE_NAME]: 'peertube'
+    })
+  })
+
+  registerInstrumentations({
+    tracerProvider: tracerProvider,
+    instrumentations: [
+      new PgInstrumentation({
+        enhancedDatabaseReporting: true
+      }),
+      new DnsInstrumentation(),
+      new HttpInstrumentation(),
+      new ExpressInstrumentation(),
+      new RedisInstrumentation({
+        dbStatementSerializer: function (cmdName, cmdArgs) {
+          return [ cmdName, ...cmdArgs ].join(' ')
+        }
+      }),
+      new FsInstrumentation()
+    ]
+  })
+
+  tracerProvider.addSpanProcessor(
+    new BatchSpanProcessor(
+      new JaegerExporter({ endpoint: CONFIG.OPEN_TELEMETRY.TRACING.JAEGER_EXPORTER.ENDPOINT })
+    )
+  )
+
+  tracerProvider.register()
+}
+
+const tracer = trace.getTracer('peertube')
+
+export {
+  registerOpentelemetryTracing,
+  tracer
+}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index e5f8b5fa2..4f711b2fa 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -24,7 +24,6 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { buildNSFWFilter } from '@server/helpers/express-utils'
 import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
 import { LiveManager } from '@server/lib/live/live-manager'
 import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
@@ -134,9 +133,9 @@ import { VideoJobInfoModel } from './video-job-info'
 import { VideoLiveModel } from './video-live'
 import { VideoPlaylistElementModel } from './video-playlist-element'
 import { VideoShareModel } from './video-share'
+import { VideoSourceModel } from './video-source'
 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
 import { VideoTagModel } from './video-tag'
-import { VideoSourceModel } from './video-source'
 
 export enum ScopeNames {
   FOR_API = 'FOR_API',
@@ -1370,11 +1369,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
   }
 
   static async getStats () {
-    const totalLocalVideos = await VideoModel.count({
-      where: {
-        remote: false
-      }
-    })
+    const serverActor = await getServerActor()
 
     let totalLocalVideoViews = await VideoModel.sum('views', {
       where: {
@@ -1385,19 +1380,26 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
     // Sequelize could return null...
     if (!totalLocalVideoViews) totalLocalVideoViews = 0
 
-    const serverActor = await getServerActor()
-
-    const { total: totalVideos } = await VideoModel.listForApi({
+    const baseOptions = {
       start: 0,
       count: 0,
       sort: '-publishedAt',
-      nsfw: buildNSFWFilter(),
+      nsfw: null,
+      isLocal: true,
       displayOnlyForFollower: {
         actorId: serverActor.id,
         orLocalVideos: true
       }
+    }
+
+    const { total: totalLocalVideos } = await VideoModel.listForApi({
+      ...baseOptions,
+
+      isLocal: true
     })
 
+    const { total: totalVideos } = await VideoModel.listForApi(baseOptions)
+
     return {
       totalLocalVideos,
       totalLocalVideoViews,
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts
index 45be107ce..78522c246 100644
--- a/server/tests/api/server/index.ts
+++ b/server/tests/api/server/index.ts
@@ -17,5 +17,6 @@ import './slow-follows'
 import './stats'
 import './tracker'
 import './no-client'
+import './open-telemetry'
 import './plugins'
 import './proxy'
diff --git a/server/tests/api/server/no-client.ts b/server/tests/api/server/no-client.ts
index 913907788..193f6c987 100644
--- a/server/tests/api/server/no-client.ts
+++ b/server/tests/api/server/no-client.ts
@@ -1,7 +1,6 @@
-import 'mocha'
 import request from 'supertest'
-import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
 import { HttpStatusCode } from '@shared/models'
+import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
 
 describe('Start and stop server without web client routes', function () {
   let server: PeerTubeServer
diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts
new file mode 100644
index 000000000..20909429f
--- /dev/null
+++ b/server/tests/api/server/open-telemetry.ts
@@ -0,0 +1,95 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { expect } from 'chai'
+import { expectLogContain, expectLogDoesNotContain, MockHTTP } from '@server/tests/shared'
+import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import { cleanupTests, createSingleServer, makeRawRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
+
+describe('Open Telemetry', function () {
+  let server: PeerTubeServer
+
+  describe('Metrics', function () {
+    const metricsUrl = 'http://localhost:9091/metrics'
+
+    it('Should not enable open telemetry metrics', async function () {
+      server = await createSingleServer(1)
+
+      let hasError = false
+      try {
+        await makeRawRequest(metricsUrl, HttpStatusCode.NOT_FOUND_404)
+      } catch (err) {
+        hasError = err.message.includes('ECONNREFUSED')
+      }
+
+      expect(hasError).to.be.true
+
+      await server.kill()
+    })
+
+    it('Should enable open telemetry metrics', async function () {
+      server = await createSingleServer(1, {
+        open_telemetry: {
+          metrics: {
+            enabled: true
+          }
+        }
+      })
+
+      const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200)
+      expect(res.text).to.contain('peertube_job_queue_total')
+
+      await server.kill()
+    })
+  })
+
+  describe('Tracing', function () {
+    let mockHTTP: MockHTTP
+    let mockPort: number
+
+    before(async function () {
+      mockHTTP = new MockHTTP()
+      mockPort = await mockHTTP.initialize()
+    })
+
+    it('Should enable open telemetry tracing', async function () {
+      server = await createSingleServer(1)
+
+      await expectLogDoesNotContain(server, 'Registering Open Telemetry tracing')
+
+      await server.kill()
+    })
+
+    it('Should enable open telemetry metrics', async function () {
+      server = await createSingleServer(1, {
+        open_telemetry: {
+          tracing: {
+            enabled: true,
+            jaeger_exporter: {
+              endpoint: 'http://localhost:' + mockPort
+            }
+          }
+        }
+      })
+
+      await expectLogContain(server, 'Registering Open Telemetry tracing')
+    })
+
+    it('Should upload a video and correctly works', async function () {
+      await setAccessTokensToServers([ server ])
+
+      const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC })
+
+      const video = await server.videos.get({ id: uuid })
+
+      expect(video.name).to.equal('video')
+    })
+
+    after(async function () {
+      await mockHTTP.terminate()
+    })
+  })
+
+  after(async function () {
+    await cleanupTests([ server ])
+  })
+})
diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts
index 33b917f31..55ebc6c3e 100644
--- a/server/tests/shared/checks.ts
+++ b/server/tests/shared/checks.ts
@@ -29,6 +29,12 @@ async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
   expect(content.toString()).to.not.contain(str)
 }
 
+async function expectLogContain (server: PeerTubeServer, str: string) {
+  const content = await server.servers.getLogContent()
+
+  expect(content.toString()).to.contain(str)
+}
+
 async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') {
   const res = await makeGetRequest({
     url,
@@ -99,5 +105,6 @@ export {
   expectNotStartWith,
   checkBadStartPagination,
   checkBadCountPagination,
-  checkBadSortPagination
+  checkBadSortPagination,
+  expectLogContain
 }
diff --git a/server/tests/shared/mock-servers/index.ts b/server/tests/shared/mock-servers/index.ts
index abf4a8203..1fa983116 100644
--- a/server/tests/shared/mock-servers/index.ts
+++ b/server/tests/shared/mock-servers/index.ts
@@ -1,5 +1,6 @@
 export * from './mock-429'
 export * from './mock-email'
+export * from './mock-http'
 export * from './mock-instances-index'
 export * from './mock-joinpeertube-versions'
 export * from './mock-object-storage'
diff --git a/server/tests/shared/mock-servers/mock-http.ts b/server/tests/shared/mock-servers/mock-http.ts
new file mode 100644
index 000000000..b7a019e07
--- /dev/null
+++ b/server/tests/shared/mock-servers/mock-http.ts
@@ -0,0 +1,23 @@
+import express from 'express'
+import { Server } from 'http'
+import { getPort, randomListen, terminateServer } from './shared'
+
+export class MockHTTP {
+  private server: Server
+
+  async initialize () {
+    const app = express()
+
+    app.get('/*', (req: express.Request, res: express.Response, next: express.NextFunction) => {
+      return res.sendStatus(200)
+    })
+
+    this.server = await randomListen(app)
+
+    return getPort(this.server)
+  }
+
+  terminate () {
+    return terminateServer(this.server)
+  }
+}
diff --git a/server/types/express.d.ts b/server/types/express.d.ts
index 27e532c31..8f8c65102 100644
--- a/server/types/express.d.ts
+++ b/server/types/express.d.ts
@@ -103,6 +103,8 @@ declare module 'express' {
     }) => void
 
     locals: {
+      requestStart: number
+
       apicache: {
         content: string | Buffer
         write: Writable['write']