aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/helpers/logger.ts101
-rw-r--r--server/initializers/config.ts16
-rw-r--r--server/initializers/constants.ts3
-rw-r--r--server/lib/activitypub/activity.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts6
-rw-r--r--server/lib/opentelemetry/metric-helpers/index.ts1
-rw-r--r--server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts186
-rw-r--r--server/lib/opentelemetry/metrics.ts111
-rw-r--r--server/lib/opentelemetry/tracing.ts81
-rw-r--r--server/models/video/video.ts24
-rw-r--r--server/tests/api/server/index.ts1
-rw-r--r--server/tests/api/server/no-client.ts3
-rw-r--r--server/tests/api/server/open-telemetry.ts95
-rw-r--r--server/tests/shared/checks.ts9
-rw-r--r--server/tests/shared/mock-servers/index.ts1
-rw-r--r--server/tests/shared/mock-servers/mock-http.ts23
-rw-r--r--server/types/express.d.ts2
17 files changed, 626 insertions, 57 deletions
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 4fbaf8a73..9625c1b33 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -1,54 +1,18 @@
1// Thanks http://tostring.it/2014/06/23/advanced-logging-with-nodejs/
2import { stat } from 'fs-extra' 1import { stat } from 'fs-extra'
3import { omit } from 'lodash' 2import { omit } from 'lodash'
4import { join } from 'path' 3import { join } from 'path'
5import { format as sqlFormat } from 'sql-formatter' 4import { format as sqlFormat } from 'sql-formatter'
6import { createLogger, format, transports } from 'winston' 5import { createLogger, format, transports } from 'winston'
7import { FileTransportOptions } from 'winston/lib/winston/transports' 6import { FileTransportOptions } from 'winston/lib/winston/transports'
7import { context } from '@opentelemetry/api'
8import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils'
8import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
9import { LOG_FILENAME } from '../initializers/constants' 10import { LOG_FILENAME } from '../initializers/constants'
10 11
11const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT 12const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
12 13
13function getLoggerReplacer () {
14 const seen = new WeakSet()
15
16 // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples
17 return (key: string, value: any) => {
18 if (key === 'cert') return 'Replaced by the logger to avoid large log message'
19
20 if (typeof value === 'object' && value !== null) {
21 if (seen.has(value)) return
22
23 seen.add(value)
24 }
25
26 if (value instanceof Set) {
27 return Array.from(value)
28 }
29
30 if (value instanceof Map) {
31 return Array.from(value.entries())
32 }
33
34 if (value instanceof Error) {
35 const error = {}
36
37 Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] })
38
39 return error
40 }
41
42 return value
43 }
44}
45
46const consoleLoggerFormat = format.printf(info => { 14const consoleLoggerFormat = format.printf(info => {
47 const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ] 15 let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2)
48
49 const obj = omit(info, ...toOmit)
50
51 let additionalInfos = JSON.stringify(obj, getLoggerReplacer(), 2)
52 16
53 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' 17 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
54 else additionalInfos = ' ' + additionalInfos 18 else additionalInfos = ' ' + additionalInfos
@@ -68,7 +32,7 @@ const consoleLoggerFormat = format.printf(info => {
68}) 32})
69 33
70const jsonLoggerFormat = format.printf(info => { 34const jsonLoggerFormat = format.printf(info => {
71 return JSON.stringify(info, getLoggerReplacer()) 35 return JSON.stringify(info, removeCyclicValues())
72}) 36})
73 37
74const timestampFormatter = format.timestamp({ 38const timestampFormatter = format.timestamp({
@@ -94,11 +58,14 @@ if (CONFIG.LOG.ROTATION.ENABLED) {
94 fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES 58 fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES
95} 59}
96 60
97const logger = buildLogger()
98
99function buildLogger (labelSuffix?: string) { 61function buildLogger (labelSuffix?: string) {
100 return createLogger({ 62 return createLogger({
101 level: CONFIG.LOG.LEVEL, 63 level: CONFIG.LOG.LEVEL,
64 defaultMeta: {
65 get traceId () { return getSpanContext(context.active())?.traceId },
66 get spanId () { return getSpanContext(context.active())?.spanId },
67 get traceFlags () { return getSpanContext(context.active())?.traceFlags }
68 },
102 format: format.combine( 69 format: format.combine(
103 labelFormatter(labelSuffix), 70 labelFormatter(labelSuffix),
104 format.splat() 71 format.splat()
@@ -118,6 +85,10 @@ function buildLogger (labelSuffix?: string) {
118 }) 85 })
119} 86}
120 87
88const logger = buildLogger()
89
90// ---------------------------------------------------------------------------
91
121function bunyanLogFactory (level: string) { 92function bunyanLogFactory (level: string) {
122 return function (...params: any[]) { 93 return function (...params: any[]) {
123 let meta = null 94 let meta = null
@@ -141,12 +112,15 @@ const bunyanLogger = {
141 level: () => { }, 112 level: () => { },
142 trace: bunyanLogFactory('debug'), 113 trace: bunyanLogFactory('debug'),
143 debug: bunyanLogFactory('debug'), 114 debug: bunyanLogFactory('debug'),
115 verbose: bunyanLogFactory('debug'),
144 info: bunyanLogFactory('info'), 116 info: bunyanLogFactory('info'),
145 warn: bunyanLogFactory('warn'), 117 warn: bunyanLogFactory('warn'),
146 error: bunyanLogFactory('error'), 118 error: bunyanLogFactory('error'),
147 fatal: bunyanLogFactory('error') 119 fatal: bunyanLogFactory('error')
148} 120}
149 121
122// ---------------------------------------------------------------------------
123
150type LoggerTagsFn = (...tags: string[]) => { tags: string[] } 124type LoggerTagsFn = (...tags: string[]) => { tags: string[] }
151function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn { 125function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
152 return (...tags: string[]) => { 126 return (...tags: string[]) => {
@@ -154,6 +128,8 @@ function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
154 } 128 }
155} 129}
156 130
131// ---------------------------------------------------------------------------
132
157async function mtimeSortFilesDesc (files: string[], basePath: string) { 133async function mtimeSortFilesDesc (files: string[], basePath: string) {
158 const promises = [] 134 const promises = []
159 const out: { file: string, mtime: number }[] = [] 135 const out: { file: string, mtime: number }[] = []
@@ -189,3 +165,44 @@ export {
189 loggerTagsFactory, 165 loggerTagsFactory,
190 bunyanLogger 166 bunyanLogger
191} 167}
168
169// ---------------------------------------------------------------------------
170
171function removeCyclicValues () {
172 const seen = new WeakSet()
173
174 // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples
175 return (key: string, value: any) => {
176 if (key === 'cert') return 'Replaced by the logger to avoid large log message'
177
178 if (typeof value === 'object' && value !== null) {
179 if (seen.has(value)) return
180
181 seen.add(value)
182 }
183
184 if (value instanceof Set) {
185 return Array.from(value)
186 }
187
188 if (value instanceof Map) {
189 return Array.from(value.entries())
190 }
191
192 if (value instanceof Error) {
193 const error = {}
194
195 Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] })
196
197 return error
198 }
199
200 return value
201 }
202}
203
204function getAdditionalInfo (info: any) {
205 const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ]
206
207 return omit(info, ...toOmit)
208}
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 754585981..0943ffe2d 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -167,6 +167,22 @@ const CONFIG = {
167 LOG_TRACKER_UNKNOWN_INFOHASH: config.get<boolean>('log.log_tracker_unknown_infohash'), 167 LOG_TRACKER_UNKNOWN_INFOHASH: config.get<boolean>('log.log_tracker_unknown_infohash'),
168 PRETTIFY_SQL: config.get<boolean>('log.prettify_sql') 168 PRETTIFY_SQL: config.get<boolean>('log.prettify_sql')
169 }, 169 },
170 OPEN_TELEMETRY: {
171 METRICS: {
172 ENABLED: config.get<boolean>('open_telemetry.metrics.enabled'),
173
174 PROMETHEUS_EXPORTER: {
175 PORT: config.get<number>('open_telemetry.metrics.prometheus_exporter.port')
176 }
177 },
178 TRACING: {
179 ENABLED: config.get<boolean>('open_telemetry.tracing.enabled'),
180
181 JAEGER_EXPORTER: {
182 ENDPOINT: config.get<string>('open_telemetry.tracing.jaeger_exporter.endpoint')
183 }
184 }
185 },
170 TRENDING: { 186 TRENDING: {
171 VIDEOS: { 187 VIDEOS: {
172 INTERVAL_DAYS: config.get<number>('trending.videos.interval_days'), 188 INTERVAL_DAYS: config.get<number>('trending.videos.interval_days'),
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index c6989c38b..e3683269c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -736,7 +736,8 @@ const MEMOIZE_TTL = {
736 INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours 736 INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours
737 VIDEO_DURATION: 1000 * 10, // 10 seconds 737 VIDEO_DURATION: 1000 * 10, // 10 seconds
738 LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute 738 LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute
739 LIVE_CHECK_SOCKET_HEALTH: 1000 * 60 // 1 minute 739 LIVE_CHECK_SOCKET_HEALTH: 1000 * 60, // 1 minute
740 GET_STATS_FOR_OPEN_TELEMETRY_METRICS: 1000 * 60 // 1 minute
740} 741}
741 742
742const MEMOIZE_LENGTH = { 743const MEMOIZE_LENGTH = {
diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts
index e6cec1ba7..ba2967ce9 100644
--- a/server/lib/activitypub/activity.ts
+++ b/server/lib/activitypub/activity.ts
@@ -1,3 +1,5 @@
1import { ActivityType } from "@shared/models"
2
1function getAPId (object: string | { id: string }) { 3function getAPId (object: string | { id: string }) {
2 if (typeof object === 'string') return object 4 if (typeof object === 'string') return object
3 5
@@ -13,8 +15,26 @@ function getDurationFromActivityStream (duration: string) {
13 return parseInt(duration.replace(/[^\d]+/, '')) 15 return parseInt(duration.replace(/[^\d]+/, ''))
14} 16}
15 17
18function buildAvailableActivities (): ActivityType[] {
19 return [
20 'Create',
21 'Update',
22 'Delete',
23 'Follow',
24 'Accept',
25 'Announce',
26 'Undo',
27 'Like',
28 'Reject',
29 'View',
30 'Dislike',
31 'Flag'
32 ]
33}
34
16export { 35export {
17 getAPId, 36 getAPId,
18 getActivityStreamDuration, 37 getActivityStreamDuration,
38 buildAvailableActivities,
19 getDurationFromActivityStream 39 getDurationFromActivityStream
20} 40}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ce24763f1..e55d2e7c2 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -285,6 +285,12 @@ class JobQueue {
285 return total 285 return total
286 } 286 }
287 287
288 async getStats () {
289 const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() }))
290
291 return Promise.all(promises)
292 }
293
288 async removeOldJobs () { 294 async removeOldJobs () {
289 for (const key of Object.keys(this.queues)) { 295 for (const key of Object.keys(this.queues)) {
290 const queue = this.queues[key] 296 const queue = this.queues[key]
diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts
new file mode 100644
index 000000000..cabb27326
--- /dev/null
+++ b/server/lib/opentelemetry/metric-helpers/index.ts
@@ -0,0 +1 @@
export * from './stats-observers-builder'
diff --git a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts
new file mode 100644
index 000000000..90b58f33d
--- /dev/null
+++ b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts
@@ -0,0 +1,186 @@
1import memoizee from 'memoizee'
2import { Meter } from '@opentelemetry/api-metrics'
3import { MEMOIZE_TTL } from '@server/initializers/constants'
4import { buildAvailableActivities } from '@server/lib/activitypub/activity'
5import { StatsManager } from '@server/lib/stat-manager'
6
7export class StatsObserverBuilder {
8
9 private readonly getInstanceStats = memoizee(() => {
10 return StatsManager.Instance.getStats()
11 }, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS })
12
13 constructor (private readonly meter: Meter) {
14
15 }
16
17 buildObservers () {
18 this.buildUserStatsObserver()
19 this.buildVideoStatsObserver()
20 this.buildCommentStatsObserver()
21 this.buildPlaylistStatsObserver()
22 this.buildChannelStatsObserver()
23 this.buildInstanceFollowsStatsObserver()
24 this.buildRedundancyStatsObserver()
25 this.buildActivityPubStatsObserver()
26 }
27
28 private buildUserStatsObserver () {
29 this.meter.createObservableGauge('peertube_users_total', {
30 description: 'Total users on the instance'
31 }).addCallback(async observableResult => {
32 const stats = await this.getInstanceStats()
33
34 observableResult.observe(stats.totalUsers)
35 })
36
37 this.meter.createObservableGauge('peertube_active_users_total', {
38 description: 'Total active users on the instance'
39 }).addCallback(async observableResult => {
40 const stats = await this.getInstanceStats()
41
42 observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' })
43 observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' })
44 observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' })
45 })
46 }
47
48 private buildChannelStatsObserver () {
49 this.meter.createObservableGauge('peertube_channels_total', {
50 description: 'Total channels on the instance'
51 }).addCallback(async observableResult => {
52 const stats = await this.getInstanceStats()
53
54 observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' })
55 })
56
57 this.meter.createObservableGauge('peertube_active_channels_total', {
58 description: 'Total active channels on the instance'
59 }).addCallback(async observableResult => {
60 const stats = await this.getInstanceStats()
61
62 observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' })
63 observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' })
64 observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' })
65 })
66 }
67
68 private buildVideoStatsObserver () {
69 this.meter.createObservableGauge('peertube_videos_total', {
70 description: 'Total videos on the instance'
71 }).addCallback(async observableResult => {
72 const stats = await this.getInstanceStats()
73
74 observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' })
75 observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' })
76 })
77
78 this.meter.createObservableGauge('peertube_video_views_total', {
79 description: 'Total video views made on the instance'
80 }).addCallback(async observableResult => {
81 const stats = await this.getInstanceStats()
82
83 observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' })
84 })
85
86 this.meter.createObservableGauge('peertube_video_bytes_total', {
87 description: 'Total bytes of videos'
88 }).addCallback(async observableResult => {
89 const stats = await this.getInstanceStats()
90
91 observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' })
92 })
93 }
94
95 private buildCommentStatsObserver () {
96 this.meter.createObservableGauge('peertube_comments_total', {
97 description: 'Total comments on the instance'
98 }).addCallback(async observableResult => {
99 const stats = await this.getInstanceStats()
100
101 observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' })
102 })
103 }
104
105 private buildPlaylistStatsObserver () {
106 this.meter.createObservableGauge('peertube_playlists_total', {
107 description: 'Total playlists on the instance'
108 }).addCallback(async observableResult => {
109 const stats = await this.getInstanceStats()
110
111 observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' })
112 })
113 }
114
115 private buildInstanceFollowsStatsObserver () {
116 this.meter.createObservableGauge('peertube_instance_followers_total', {
117 description: 'Total followers of the instance'
118 }).addCallback(async observableResult => {
119 const stats = await this.getInstanceStats()
120
121 observableResult.observe(stats.totalInstanceFollowers)
122 })
123
124 this.meter.createObservableGauge('peertube_instance_following_total', {
125 description: 'Total following of the instance'
126 }).addCallback(async observableResult => {
127 const stats = await this.getInstanceStats()
128
129 observableResult.observe(stats.totalInstanceFollowing)
130 })
131 }
132
133 private buildRedundancyStatsObserver () {
134 this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', {
135 description: 'Total redundancy used of the instance'
136 }).addCallback(async observableResult => {
137 const stats = await this.getInstanceStats()
138
139 for (const r of stats.videosRedundancy) {
140 observableResult.observe(r.totalUsed, { strategy: r.strategy })
141 }
142 })
143
144 this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', {
145 description: 'Total redundancy available of the instance'
146 }).addCallback(async observableResult => {
147 const stats = await this.getInstanceStats()
148
149 for (const r of stats.videosRedundancy) {
150 observableResult.observe(r.totalSize, { strategy: r.strategy })
151 }
152 })
153 }
154
155 private buildActivityPubStatsObserver () {
156 const availableActivities = buildAvailableActivities()
157
158 this.meter.createObservableGauge('peertube_ap_inbox_success_total', {
159 description: 'Total inbox messages processed with success'
160 }).addCallback(async observableResult => {
161 const stats = await this.getInstanceStats()
162
163 for (const type of availableActivities) {
164 observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type })
165 }
166 })
167
168 this.meter.createObservableGauge('peertube_ap_inbox_error_total', {
169 description: 'Total inbox messages processed with error'
170 }).addCallback(async observableResult => {
171 const stats = await this.getInstanceStats()
172
173 for (const type of availableActivities) {
174 observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type })
175 }
176 })
177
178 this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', {
179 description: 'Total inbox messages waiting for being processed'
180 }).addCallback(async observableResult => {
181 const stats = await this.getInstanceStats()
182
183 observableResult.observe(stats.totalActivityPubMessagesWaiting)
184 })
185 }
186}
diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts
new file mode 100644
index 000000000..ca0aae8e7
--- /dev/null
+++ b/server/lib/opentelemetry/metrics.ts
@@ -0,0 +1,111 @@
1import { Application, Request, Response } from 'express'
2import { Meter, metrics } from '@opentelemetry/api-metrics'
3import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
4import { MeterProvider } from '@opentelemetry/sdk-metrics-base'
5import { logger } from '@server/helpers/logger'
6import { CONFIG } from '@server/initializers/config'
7import { JobQueue } from '../job-queue'
8import { StatsObserverBuilder } from './metric-helpers'
9
10class OpenTelemetryMetrics {
11
12 private static instance: OpenTelemetryMetrics
13
14 private meter: Meter
15
16 private onRequestDuration: (req: Request, res: Response) => void
17
18 private constructor () {}
19
20 init (app: Application) {
21 if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return
22
23 app.use((req, res, next) => {
24 res.once('finish', () => {
25 if (!this.onRequestDuration) return
26
27 this.onRequestDuration(req as Request, res as Response)
28 })
29
30 next()
31 })
32 }
33
34 registerMetrics () {
35 if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return
36
37 logger.info('Registering Open Telemetry metrics')
38
39 const provider = new MeterProvider()
40
41 provider.addMetricReader(new PrometheusExporter({ port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT }))
42
43 metrics.setGlobalMeterProvider(provider)
44
45 this.meter = metrics.getMeter('default')
46
47 this.buildMemoryObserver()
48 this.buildRequestObserver()
49 this.buildJobQueueObserver()
50
51 const statsObserverBuilder = new StatsObserverBuilder(this.meter)
52 statsObserverBuilder.buildObservers()
53 }
54
55 private buildMemoryObserver () {
56 this.meter.createObservableGauge('nodejs_memory_usage_bytes', {
57 description: 'Memory'
58 }).addCallback(observableResult => {
59 const current = process.memoryUsage()
60
61 observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' })
62 observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' })
63 observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' })
64 observableResult.observe(current.external, { memoryType: 'external' })
65 observableResult.observe(current.rss, { memoryType: 'rss' })
66 })
67 }
68
69 private buildJobQueueObserver () {
70 this.meter.createObservableGauge('peertube_job_queue_total', {
71 description: 'Total jobs in the PeerTube job queue'
72 }).addCallback(async observableResult => {
73 const stats = await JobQueue.Instance.getStats()
74
75 for (const { jobType, counts } of stats) {
76 for (const state of Object.keys(counts)) {
77 observableResult.observe(counts[state], { jobType, state })
78 }
79 }
80 })
81 }
82
83 private buildRequestObserver () {
84 const requestDuration = this.meter.createHistogram('http_request_duration_ms', {
85 unit: 'milliseconds',
86 description: 'Duration of HTTP requests in ms'
87 })
88
89 this.onRequestDuration = (req: Request, res: Response) => {
90 const duration = Date.now() - res.locals.requestStart
91
92 requestDuration.record(duration, {
93 path: this.buildRequestPath(req.originalUrl),
94 method: req.method,
95 statusCode: res.statusCode + ''
96 })
97 }
98 }
99
100 private buildRequestPath (path: string) {
101 return path.split('?')[0]
102 }
103
104 static get Instance () {
105 return this.instance || (this.instance = new this())
106 }
107}
108
109export {
110 OpenTelemetryMetrics
111}
diff --git a/server/lib/opentelemetry/tracing.ts b/server/lib/opentelemetry/tracing.ts
new file mode 100644
index 000000000..5358d04de
--- /dev/null
+++ b/server/lib/opentelemetry/tracing.ts
@@ -0,0 +1,81 @@
1import { diag, DiagLogLevel, trace } from '@opentelemetry/api'
2import { JaegerExporter } from '@opentelemetry/exporter-jaeger'
3import { registerInstrumentations } from '@opentelemetry/instrumentation'
4import { DnsInstrumentation } from '@opentelemetry/instrumentation-dns'
5import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express'
6import FsInstrumentation from '@opentelemetry/instrumentation-fs'
7import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
8import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'
9import { RedisInstrumentation } from '@opentelemetry/instrumentation-redis-4'
10import { Resource } from '@opentelemetry/resources'
11import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'
12import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'
13import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'
14import { logger } from '@server/helpers/logger'
15import { CONFIG } from '@server/initializers/config'
16
17function registerOpentelemetryTracing () {
18 if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) return
19
20 logger.info('Registering Open Telemetry tracing')
21
22 const customLogger = (level: string) => {
23 return (message: string, ...args: unknown[]) => {
24 let fullMessage = message
25
26 for (const arg of args) {
27 if (typeof arg === 'string') fullMessage += arg
28 else break
29 }
30
31 logger[level](fullMessage)
32 }
33 }
34
35 diag.setLogger({
36 error: customLogger('error'),
37 warn: customLogger('warn'),
38 info: customLogger('info'),
39 debug: customLogger('debug'),
40 verbose: customLogger('verbose')
41 }, DiagLogLevel.INFO)
42
43 const tracerProvider = new NodeTracerProvider({
44 resource: new Resource({
45 [SemanticResourceAttributes.SERVICE_NAME]: 'peertube'
46 })
47 })
48
49 registerInstrumentations({
50 tracerProvider: tracerProvider,
51 instrumentations: [
52 new PgInstrumentation({
53 enhancedDatabaseReporting: true
54 }),
55 new DnsInstrumentation(),
56 new HttpInstrumentation(),
57 new ExpressInstrumentation(),
58 new RedisInstrumentation({
59 dbStatementSerializer: function (cmdName, cmdArgs) {
60 return [ cmdName, ...cmdArgs ].join(' ')
61 }
62 }),
63 new FsInstrumentation()
64 ]
65 })
66
67 tracerProvider.addSpanProcessor(
68 new BatchSpanProcessor(
69 new JaegerExporter({ endpoint: CONFIG.OPEN_TELEMETRY.TRACING.JAEGER_EXPORTER.ENDPOINT })
70 )
71 )
72
73 tracerProvider.register()
74}
75
76const tracer = trace.getTracer('peertube')
77
78export {
79 registerOpentelemetryTracing,
80 tracer
81}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index e5f8b5fa2..4f711b2fa 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -24,7 +24,6 @@ import {
24 Table, 24 Table,
25 UpdatedAt 25 UpdatedAt
26} from 'sequelize-typescript' 26} from 'sequelize-typescript'
27import { buildNSFWFilter } from '@server/helpers/express-utils'
28import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' 27import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
29import { LiveManager } from '@server/lib/live/live-manager' 28import { LiveManager } from '@server/lib/live/live-manager'
30import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' 29import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
@@ -134,9 +133,9 @@ import { VideoJobInfoModel } from './video-job-info'
134import { VideoLiveModel } from './video-live' 133import { VideoLiveModel } from './video-live'
135import { VideoPlaylistElementModel } from './video-playlist-element' 134import { VideoPlaylistElementModel } from './video-playlist-element'
136import { VideoShareModel } from './video-share' 135import { VideoShareModel } from './video-share'
136import { VideoSourceModel } from './video-source'
137import { VideoStreamingPlaylistModel } from './video-streaming-playlist' 137import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
138import { VideoTagModel } from './video-tag' 138import { VideoTagModel } from './video-tag'
139import { VideoSourceModel } from './video-source'
140 139
141export enum ScopeNames { 140export enum ScopeNames {
142 FOR_API = 'FOR_API', 141 FOR_API = 'FOR_API',
@@ -1370,11 +1369,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1370 } 1369 }
1371 1370
1372 static async getStats () { 1371 static async getStats () {
1373 const totalLocalVideos = await VideoModel.count({ 1372 const serverActor = await getServerActor()
1374 where: {
1375 remote: false
1376 }
1377 })
1378 1373
1379 let totalLocalVideoViews = await VideoModel.sum('views', { 1374 let totalLocalVideoViews = await VideoModel.sum('views', {
1380 where: { 1375 where: {
@@ -1385,19 +1380,26 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1385 // Sequelize could return null... 1380 // Sequelize could return null...
1386 if (!totalLocalVideoViews) totalLocalVideoViews = 0 1381 if (!totalLocalVideoViews) totalLocalVideoViews = 0
1387 1382
1388 const serverActor = await getServerActor() 1383 const baseOptions = {
1389
1390 const { total: totalVideos } = await VideoModel.listForApi({
1391 start: 0, 1384 start: 0,
1392 count: 0, 1385 count: 0,
1393 sort: '-publishedAt', 1386 sort: '-publishedAt',
1394 nsfw: buildNSFWFilter(), 1387 nsfw: null,
1388 isLocal: true,
1395 displayOnlyForFollower: { 1389 displayOnlyForFollower: {
1396 actorId: serverActor.id, 1390 actorId: serverActor.id,
1397 orLocalVideos: true 1391 orLocalVideos: true
1398 } 1392 }
1393 }
1394
1395 const { total: totalLocalVideos } = await VideoModel.listForApi({
1396 ...baseOptions,
1397
1398 isLocal: true
1399 }) 1399 })
1400 1400
1401 const { total: totalVideos } = await VideoModel.listForApi(baseOptions)
1402
1401 return { 1403 return {
1402 totalLocalVideos, 1404 totalLocalVideos,
1403 totalLocalVideoViews, 1405 totalLocalVideoViews,
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts
index 45be107ce..78522c246 100644
--- a/server/tests/api/server/index.ts
+++ b/server/tests/api/server/index.ts
@@ -17,5 +17,6 @@ import './slow-follows'
17import './stats' 17import './stats'
18import './tracker' 18import './tracker'
19import './no-client' 19import './no-client'
20import './open-telemetry'
20import './plugins' 21import './plugins'
21import './proxy' 22import './proxy'
diff --git a/server/tests/api/server/no-client.ts b/server/tests/api/server/no-client.ts
index 913907788..193f6c987 100644
--- a/server/tests/api/server/no-client.ts
+++ b/server/tests/api/server/no-client.ts
@@ -1,7 +1,6 @@
1import 'mocha'
2import request from 'supertest' 1import request from 'supertest'
3import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
4import { HttpStatusCode } from '@shared/models' 2import { HttpStatusCode } from '@shared/models'
3import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
5 4
6describe('Start and stop server without web client routes', function () { 5describe('Start and stop server without web client routes', function () {
7 let server: PeerTubeServer 6 let server: PeerTubeServer
diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts
new file mode 100644
index 000000000..20909429f
--- /dev/null
+++ b/server/tests/api/server/open-telemetry.ts
@@ -0,0 +1,95 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { expectLogContain, expectLogDoesNotContain, MockHTTP } from '@server/tests/shared'
5import { HttpStatusCode, VideoPrivacy } from '@shared/models'
6import { cleanupTests, createSingleServer, makeRawRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
7
8describe('Open Telemetry', function () {
9 let server: PeerTubeServer
10
11 describe('Metrics', function () {
12 const metricsUrl = 'http://localhost:9091/metrics'
13
14 it('Should not enable open telemetry metrics', async function () {
15 server = await createSingleServer(1)
16
17 let hasError = false
18 try {
19 await makeRawRequest(metricsUrl, HttpStatusCode.NOT_FOUND_404)
20 } catch (err) {
21 hasError = err.message.includes('ECONNREFUSED')
22 }
23
24 expect(hasError).to.be.true
25
26 await server.kill()
27 })
28
29 it('Should enable open telemetry metrics', async function () {
30 server = await createSingleServer(1, {
31 open_telemetry: {
32 metrics: {
33 enabled: true
34 }
35 }
36 })
37
38 const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200)
39 expect(res.text).to.contain('peertube_job_queue_total')
40
41 await server.kill()
42 })
43 })
44
45 describe('Tracing', function () {
46 let mockHTTP: MockHTTP
47 let mockPort: number
48
49 before(async function () {
50 mockHTTP = new MockHTTP()
51 mockPort = await mockHTTP.initialize()
52 })
53
54 it('Should enable open telemetry tracing', async function () {
55 server = await createSingleServer(1)
56
57 await expectLogDoesNotContain(server, 'Registering Open Telemetry tracing')
58
59 await server.kill()
60 })
61
62 it('Should enable open telemetry metrics', async function () {
63 server = await createSingleServer(1, {
64 open_telemetry: {
65 tracing: {
66 enabled: true,
67 jaeger_exporter: {
68 endpoint: 'http://localhost:' + mockPort
69 }
70 }
71 }
72 })
73
74 await expectLogContain(server, 'Registering Open Telemetry tracing')
75 })
76
77 it('Should upload a video and correctly works', async function () {
78 await setAccessTokensToServers([ server ])
79
80 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC })
81
82 const video = await server.videos.get({ id: uuid })
83
84 expect(video.name).to.equal('video')
85 })
86
87 after(async function () {
88 await mockHTTP.terminate()
89 })
90 })
91
92 after(async function () {
93 await cleanupTests([ server ])
94 })
95})
diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts
index 33b917f31..55ebc6c3e 100644
--- a/server/tests/shared/checks.ts
+++ b/server/tests/shared/checks.ts
@@ -29,6 +29,12 @@ async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
29 expect(content.toString()).to.not.contain(str) 29 expect(content.toString()).to.not.contain(str)
30} 30}
31 31
32async function expectLogContain (server: PeerTubeServer, str: string) {
33 const content = await server.servers.getLogContent()
34
35 expect(content.toString()).to.contain(str)
36}
37
32async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { 38async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') {
33 const res = await makeGetRequest({ 39 const res = await makeGetRequest({
34 url, 40 url,
@@ -99,5 +105,6 @@ export {
99 expectNotStartWith, 105 expectNotStartWith,
100 checkBadStartPagination, 106 checkBadStartPagination,
101 checkBadCountPagination, 107 checkBadCountPagination,
102 checkBadSortPagination 108 checkBadSortPagination,
109 expectLogContain
103} 110}
diff --git a/server/tests/shared/mock-servers/index.ts b/server/tests/shared/mock-servers/index.ts
index abf4a8203..1fa983116 100644
--- a/server/tests/shared/mock-servers/index.ts
+++ b/server/tests/shared/mock-servers/index.ts
@@ -1,5 +1,6 @@
1export * from './mock-429' 1export * from './mock-429'
2export * from './mock-email' 2export * from './mock-email'
3export * from './mock-http'
3export * from './mock-instances-index' 4export * from './mock-instances-index'
4export * from './mock-joinpeertube-versions' 5export * from './mock-joinpeertube-versions'
5export * from './mock-object-storage' 6export * from './mock-object-storage'
diff --git a/server/tests/shared/mock-servers/mock-http.ts b/server/tests/shared/mock-servers/mock-http.ts
new file mode 100644
index 000000000..b7a019e07
--- /dev/null
+++ b/server/tests/shared/mock-servers/mock-http.ts
@@ -0,0 +1,23 @@
1import express from 'express'
2import { Server } from 'http'
3import { getPort, randomListen, terminateServer } from './shared'
4
5export class MockHTTP {
6 private server: Server
7
8 async initialize () {
9 const app = express()
10
11 app.get('/*', (req: express.Request, res: express.Response, next: express.NextFunction) => {
12 return res.sendStatus(200)
13 })
14
15 this.server = await randomListen(app)
16
17 return getPort(this.server)
18 }
19
20 terminate () {
21 return terminateServer(this.server)
22 }
23}
diff --git a/server/types/express.d.ts b/server/types/express.d.ts
index 27e532c31..8f8c65102 100644
--- a/server/types/express.d.ts
+++ b/server/types/express.d.ts
@@ -103,6 +103,8 @@ declare module 'express' {
103 }) => void 103 }) => void
104 104
105 locals: { 105 locals: {
106 requestStart: number
107
106 apicache: { 108 apicache: {
107 content: string | Buffer 109 content: string | Buffer
108 write: Writable['write'] 110 write: Writable['write']