From 42b40636991b97fe818007fab19091764fc5db73 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 15 Jul 2022 15:30:14 +0200 Subject: Add ability for client to create server logs --- server/controllers/api/server/logs.ts | 41 +++++++++++++++++--- server/helpers/custom-validators/logs.ts | 36 ++++++++++++++++-- server/initializers/config.ts | 7 +++- server/initializers/constants.ts | 6 +++ server/middlewares/validators/logs.ts | 52 +++++++++++++++++++++++-- server/tests/api/check-params/logs.ts | 59 ++++++++++++++++++++++++++++- server/tests/api/server/logs.ts | 65 ++++++++++++++++++++++++++++++++ 7 files changed, 251 insertions(+), 15 deletions(-) (limited to 'server') diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts index 8aa4b7190..ed0aa6e8e 100644 --- a/server/controllers/api/server/logs.ts +++ b/server/controllers/api/server/logs.ts @@ -3,15 +3,29 @@ import { readdir, readFile } from 'fs-extra' import { join } from 'path' import { isArray } from '@server/helpers/custom-validators/misc' import { logger, mtimeSortFilesDesc } from '@server/helpers/logger' -import { LogLevel } from '../../../../shared/models/server/log-level.type' +import { pick } from '@shared/core-utils' +import { ClientLogCreate, HttpStatusCode } from '@shared/models' +import { ServerLogLevel } from '../../../../shared/models/server/server-log-level.type' import { UserRight } from '../../../../shared/models/users' import { CONFIG } from '../../../initializers/config' import { AUDIT_LOG_FILENAME, LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants' -import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' -import { getAuditLogsValidator, getLogsValidator } from '../../../middlewares/validators/logs' +import { asyncMiddleware, authenticate, buildRateLimiter, ensureUserHasRight, optionalAuthenticate } from '../../../middlewares' +import { createClientLogValidator, getAuditLogsValidator, getLogsValidator } from '../../../middlewares/validators/logs' + +const createClientLogRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.RECEIVE_CLIENT_LOG.WINDOW_MS, + max: CONFIG.RATES_LIMIT.RECEIVE_CLIENT_LOG.MAX +}) const logsRouter = express.Router() +logsRouter.post('/logs/client', + createClientLogRateLimiter, + optionalAuthenticate, + createClientLogValidator, + createClientLog +) + logsRouter.get('/logs', authenticate, ensureUserHasRight(UserRight.MANAGE_LOGS), @@ -34,6 +48,21 @@ export { // --------------------------------------------------------------------------- +function createClientLog (req: express.Request, res: express.Response) { + const logInfo = req.body as ClientLogCreate + + const meta = { + tags: [ 'client' ], + username: res.locals.oauth?.token?.User?.username, + + ...pick(logInfo, [ 'userAgent', 'stackTrace', 'meta', 'url' ]) + } + + logger.log(logInfo.level, `Client log: ${logInfo.message}`, meta) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + const auditLogNameFilter = generateLogNameFilter(AUDIT_LOG_FILENAME) async function getAuditLogs (req: express.Request, res: express.Response) { const output = await generateOutput({ @@ -63,7 +92,7 @@ async function generateOutput (options: { startDateQuery: string endDateQuery?: string - level: LogLevel + level: ServerLogLevel nameFilter: RegExp tagsOneOf?: string[] }) { @@ -104,7 +133,7 @@ async function getOutputFromFile (options: { path: string startDate: Date endDate: Date - level: LogLevel + level: ServerLogLevel currentSize: number tagsOneOf: Set }) { @@ -116,7 +145,7 @@ async function getOutputFromFile (options: { let logTime: number - const logsLevel: { [ id in LogLevel ]: number } = { + const logsLevel: { [ id in ServerLogLevel ]: number } = { audit: -1, debug: 0, info: 1, diff --git a/server/helpers/custom-validators/logs.ts b/server/helpers/custom-validators/logs.ts index 0f266ed3b..41d45cbb2 100644 --- a/server/helpers/custom-validators/logs.ts +++ b/server/helpers/custom-validators/logs.ts @@ -1,14 +1,42 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' +import { ClientLogLevel, ServerLogLevel } from '@shared/models' import { exists } from './misc' -import { LogLevel } from '../../../shared/models/server/log-level.type' -const logLevels: LogLevel[] = [ 'debug', 'info', 'warn', 'error' ] +const serverLogLevels: Set = new Set([ 'debug', 'info', 'warn', 'error' ]) +const clientLogLevels: Set = new Set([ 'warn', 'error' ]) function isValidLogLevel (value: any) { - return exists(value) && logLevels.includes(value) + return exists(value) && serverLogLevels.has(value) +} + +function isValidClientLogMessage (value: any) { + return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_MESSAGE) +} + +function isValidClientLogLevel (value: any) { + return exists(value) && clientLogLevels.has(value) +} + +function isValidClientLogStackTrace (value: any) { + return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_STACK_TRACE) +} + +function isValidClientLogMeta (value: any) { + return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_META) +} + +function isValidClientLogUserAgent (value: any) { + return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_USER_AGENT) } // --------------------------------------------------------------------------- export { - isValidLogLevel + isValidLogLevel, + isValidClientLogMessage, + isValidClientLogStackTrace, + isValidClientLogMeta, + isValidClientLogLevel, + isValidClientLogUserAgent } diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 0943ffe2d..ba0f756ef 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -149,6 +149,10 @@ const CONFIG = { WINDOW_MS: parseDurationToMs(config.get('rates_limit.login.window')), MAX: config.get('rates_limit.login.max') }, + RECEIVE_CLIENT_LOG: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.receive_client_log.window')), + MAX: config.get('rates_limit.receive_client_log.max') + }, ASK_SEND_EMAIL: { WINDOW_MS: parseDurationToMs(config.get('rates_limit.ask_send_email.window')), MAX: config.get('rates_limit.ask_send_email.max') @@ -165,7 +169,8 @@ const CONFIG = { ANONYMIZE_IP: config.get('log.anonymize_ip'), LOG_PING_REQUESTS: config.get('log.log_ping_requests'), LOG_TRACKER_UNKNOWN_INFOHASH: config.get('log.log_tracker_unknown_infohash'), - PRETTIFY_SQL: config.get('log.prettify_sql') + PRETTIFY_SQL: config.get('log.prettify_sql'), + ACCEPT_CLIENT_LOG: config.get('log.accept_client_log') }, OPEN_TELEMETRY: { METRICS: { diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 009f878fc..8cb4d5f4a 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -365,6 +365,12 @@ const CONSTRAINTS_FIELDS = { VIDEO_STUDIO: { TASKS: { min: 1, max: 10 }, // Number of tasks CUT_TIME: { min: 0 } // Value + }, + LOGS: { + CLIENT_MESSAGE: { min: 1, max: 1000 }, // Length + CLIENT_STACK_TRACE: { min: 1, max: 5000 }, // Length + CLIENT_META: { min: 1, max: 5000 }, // Length + CLIENT_USER_AGENT: { min: 1, max: 200 } // Length } } diff --git a/server/middlewares/validators/logs.ts b/server/middlewares/validators/logs.ts index 901d8ca64..324ba6915 100644 --- a/server/middlewares/validators/logs.ts +++ b/server/middlewares/validators/logs.ts @@ -1,11 +1,56 @@ import express from 'express' -import { query } from 'express-validator' +import { body, query } from 'express-validator' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' import { isStringArray } from '@server/helpers/custom-validators/search' -import { isValidLogLevel } from '../../helpers/custom-validators/logs' +import { CONFIG } from '@server/initializers/config' +import { HttpStatusCode } from '@shared/models' +import { + isValidClientLogLevel, + isValidClientLogMessage, + isValidClientLogMeta, + isValidClientLogStackTrace, + isValidClientLogUserAgent, + isValidLogLevel +} from '../../helpers/custom-validators/logs' import { isDateValid, toArray } from '../../helpers/custom-validators/misc' import { logger } from '../../helpers/logger' import { areValidationErrors } from './shared' +const createClientLogValidator = [ + body('message') + .custom(isValidClientLogMessage).withMessage('Should have a valid log message'), + + body('url') + .custom(isUrlValid).withMessage('Should have a valid log url'), + + body('level') + .custom(isValidClientLogLevel).withMessage('Should have a valid log message'), + + body('stackTrace') + .optional() + .custom(isValidClientLogStackTrace).withMessage('Should have a valid log stack trace'), + + body('meta') + .optional() + .custom(isValidClientLogMeta).withMessage('Should have a valid log meta'), + + body('userAgent') + .optional() + .custom(isValidClientLogUserAgent).withMessage('Should have a valid log user agent'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking createClientLogValidator parameters.', { parameters: req.query }) + + if (CONFIG.LOG.ACCEPT_CLIENT_LOG !== true) { + return res.sendStatus(HttpStatusCode.FORBIDDEN_403) + } + + if (areValidationErrors(req, res)) return + + return next() + } +] + const getLogsValidator = [ query('startDate') .custom(isDateValid).withMessage('Should have a start date that conforms to ISO 8601'), @@ -49,5 +94,6 @@ const getAuditLogsValidator = [ export { getLogsValidator, - getAuditLogsValidator + getAuditLogsValidator, + createClientLogValidator } diff --git a/server/tests/api/check-params/logs.ts b/server/tests/api/check-params/logs.ts index 970671c15..fa67408b7 100644 --- a/server/tests/api/check-params/logs.ts +++ b/server/tests/api/check-params/logs.ts @@ -1,8 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' -import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' +import { expect } from 'chai' import { HttpStatusCode } from '@shared/models' +import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' describe('Test logs API validators', function () { const path = '/api/v1/server/logs' @@ -95,6 +96,62 @@ describe('Test logs API validators', function () { }) }) + describe('When creating client logs', function () { + const base = { + level: 'warn' as 'warn', + message: 'my super message', + url: 'https://example.com/toto' + } + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + it('Should fail with an invalid level', async function () { + await server.logs.createLogClient({ payload: { ...base, level: '' as any }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, level: undefined }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, level: 'toto' as any }, expectedStatus }) + }) + + it('Should fail with an invalid message', async function () { + await server.logs.createLogClient({ payload: { ...base, message: undefined }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, message: '' }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, message: 'm'.repeat(2500) }, expectedStatus }) + }) + + it('Should fail with an invalid url', async function () { + await server.logs.createLogClient({ payload: { ...base, url: undefined }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, url: 'toto' }, expectedStatus }) + }) + + it('Should fail with an invalid stackTrace', async function () { + await server.logs.createLogClient({ payload: { ...base, stackTrace: 's'.repeat(10000) }, expectedStatus }) + }) + + it('Should fail with an invalid userAgent', async function () { + await server.logs.createLogClient({ payload: { ...base, userAgent: 's'.repeat(500) }, expectedStatus }) + }) + + it('Should fail with an invalid meta', async function () { + await server.logs.createLogClient({ payload: { ...base, meta: 's'.repeat(10000) }, expectedStatus }) + }) + + it('Should succeed with the correct params', async function () { + await server.logs.createLogClient({ payload: { ...base, stackTrace: 'stackTrace', meta: '{toto}', userAgent: 'userAgent' } }) + }) + + it('Should rate limit log creation', async function () { + let fail = false + + for (let i = 0; i < 10; i++) { + try { + await server.logs.createLogClient({ token: null, payload: base }) + } catch { + fail = true + } + } + + expect(fail).to.be.true + }) + }) + after(async function () { await cleanupTests([ server ]) }) diff --git a/server/tests/api/server/logs.ts b/server/tests/api/server/logs.ts index 697f10337..ed7555fd7 100644 --- a/server/tests/api/server/logs.ts +++ b/server/tests/api/server/logs.ts @@ -2,6 +2,7 @@ import 'mocha' import * as chai from 'chai' +import { HttpStatusCode } from '@shared/models' import { cleanupTests, createSingleServer, @@ -198,6 +199,70 @@ describe('Test logs', function () { }) }) + describe('When creating log from the client', function () { + + it('Should create a warn client log', async function () { + const now = new Date() + + await server.logs.createLogClient({ + payload: { + level: 'warn', + url: 'http://example.com', + message: 'my super client message' + }, + token: null + }) + + const body = await logsCommand.getLogs({ startDate: now }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('my super client message')).to.be.true + }) + + it('Should create an error authenticated client log', async function () { + const now = new Date() + + await server.logs.createLogClient({ + payload: { + url: 'https://example.com/page1', + level: 'error', + message: 'my super client message 2', + userAgent: 'super user agent', + meta: '{hello}', + stackTrace: 'super stack trace' + } + }) + + const body = await logsCommand.getLogs({ startDate: now }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('my super client message 2')).to.be.true + expect(logsString.includes('super user agent')).to.be.true + expect(logsString.includes('super stack trace')).to.be.true + expect(logsString.includes('{hello}')).to.be.true + expect(logsString.includes('https://example.com/page1')).to.be.true + }) + + it('Should refuse to create client logs', async function () { + await server.kill() + + await server.run({ + log: { + accept_client_log: false + } + }) + + await server.logs.createLogClient({ + payload: { + level: 'warn', + url: 'http://example.com', + message: 'my super client message' + }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + after(async function () { await cleanupTests([ server ]) }) -- cgit v1.2.3