From 566c125d6eee3bd907404523d94e1e0b5e403a46 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Dec 2019 14:14:01 +0100 Subject: Serve audit logs to client --- server/controllers/api/server/logs.ts | 57 +++++++++++-- server/helpers/audit-logger.ts | 3 +- server/helpers/logger.ts | 3 +- server/initializers/constants.ts | 4 + server/middlewares/validators/logs.ts | 19 ++++- server/tests/api/server/logs.ts | 145 ++++++++++++++++++++++------------ 6 files changed, 173 insertions(+), 58 deletions(-) (limited to 'server') diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts index e9d1f2efd..a0ca21cd5 100644 --- a/server/controllers/api/server/logs.ts +++ b/server/controllers/api/server/logs.ts @@ -3,11 +3,12 @@ import { UserRight } from '../../../../shared/models/users' import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' import { mtimeSortFilesDesc } from '../../../../shared/core-utils/logs/logs' import { readdir, readFile } from 'fs-extra' -import { MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants' +import { AUDIT_LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS, LOG_FILENAME } from '../../../initializers/constants' import { join } from 'path' -import { getLogsValidator } from '../../../middlewares/validators/logs' +import { getAuditLogsValidator, getLogsValidator } from '../../../middlewares/validators/logs' import { LogLevel } from '../../../../shared/models/server/log-level.type' import { CONFIG } from '../../../initializers/config' +import { logger } from '@server/helpers/logger' const logsRouter = express.Router() @@ -18,6 +19,13 @@ logsRouter.get('/logs', asyncMiddleware(getLogs) ) +logsRouter.get('/audit-logs', + authenticate, + ensureUserHasRight(UserRight.MANAGE_LOGS), + getAuditLogsValidator, + asyncMiddleware(getAuditLogs) +) + // --------------------------------------------------------------------------- export { @@ -26,18 +34,50 @@ export { // --------------------------------------------------------------------------- +const auditLogNameFilter = generateLogNameFilter(AUDIT_LOG_FILENAME) +async function getAuditLogs (req: express.Request, res: express.Response) { + const output = await generateOutput({ + startDateQuery: req.query.startDate, + endDateQuery: req.query.endDate, + level: 'audit', + nameFilter: auditLogNameFilter + }) + + return res.json(output).end() +} + +const logNameFilter = generateLogNameFilter(LOG_FILENAME) async function getLogs (req: express.Request, res: express.Response) { + const output = await generateOutput({ + startDateQuery: req.query.startDate, + endDateQuery: req.query.endDate, + level: req.query.level || 'info', + nameFilter: logNameFilter + }) + + return res.json(output).end() +} + +async function generateOutput (options: { + startDateQuery: string, + endDateQuery?: string, + level: LogLevel, + nameFilter: RegExp +}) { + const { startDateQuery, level, nameFilter } = options + const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR) let currentSize = 0 - const startDate = new Date(req.query.startDate) - const endDate = req.query.endDate ? new Date(req.query.endDate) : new Date() - const level: LogLevel = req.query.level || 'info' + const startDate = new Date(startDateQuery) + const endDate = options.endDateQuery ? new Date(options.endDateQuery) : new Date() let output: string[] = [] for (const meta of sortedLogFiles) { + if (nameFilter.exec(meta.file) === null) continue + const path = join(CONFIG.STORAGE.LOG_DIR, meta.file) const result = await getOutputFromFile(path, startDate, endDate, level, currentSize) @@ -49,7 +89,7 @@ async function getLogs (req: express.Request, res: express.Response) { if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS || (result.logTime && result.logTime < startDate.getTime())) break } - return res.json(output).end() + return output } async function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) { @@ -58,6 +98,7 @@ async function getOutputFromFile (path: string, startDate: Date, endDate: Date, let logTime: number const logsLevel: { [ id in LogLevel ]: number } = { + audit: -1, debug: 0, info: 1, warn: 2, @@ -93,3 +134,7 @@ async function getOutputFromFile (path: string, startDate: Date, endDate: Date, return { currentSize, output: output.reverse(), logTime } } + +function generateLogNameFilter (baseName: string) { + return new RegExp('^' + baseName.replace(/\.log$/, '') + '\d*.log$') +} diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index f536da439..9b258dc3a 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts @@ -9,6 +9,7 @@ import { User, VideoAbuse, VideoChannel, VideoDetails, VideoImport } from '../.. import { VideoComment } from '../../shared/models/videos/video-comment.model' import { CustomConfig } from '../../shared/models/server/custom-config.model' import { CONFIG } from '../initializers/config' +import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' function getAuditIdFromRes (res: express.Response) { return res.locals.oauth.token.User.username @@ -29,7 +30,7 @@ const auditLogger = winston.createLogger({ levels: { audit: 0 }, transports: [ new winston.transports.File({ - filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube-audit.log'), + filename: path.join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME), level: 'audit', maxsize: 5242880, maxFiles: 5, diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index d21746963..c2ff2bae6 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -5,6 +5,7 @@ import * as winston from 'winston' import { FileTransportOptions } from 'winston/lib/winston/transports' import { CONFIG } from '../initializers/config' import { omit } from 'lodash' +import { LOG_FILENAME } from '@server/initializers/constants' const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT @@ -58,7 +59,7 @@ const labelFormatter = winston.format.label({ }) const fileLoggerOptions: FileTransportOptions = { - filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube.log'), + filename: path.join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME), handleExceptions: true, format: winston.format.combine( winston.format.timestamp(), diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index af70e7b88..bdabe7f66 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -603,6 +603,8 @@ const FEEDS = { } const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000 +const LOG_FILENAME = 'peertube.log' +const AUDIT_LOG_FILENAME = 'peertube-audit.log' // --------------------------------------------------------------------------- @@ -684,6 +686,7 @@ export { BCRYPT_SALT_SIZE, TRACKER_RATE_LIMITS, FILES_CACHE, + LOG_FILENAME, CONSTRAINTS_FIELDS, EMBED_SIZE, REDUNDANCY, @@ -693,6 +696,7 @@ export { OAUTH_LIFETIME, CUSTOM_HTML_TAG_COMMENTS, BROADCAST_CONCURRENCY, + AUDIT_LOG_FILENAME, PAGINATION, ACTOR_FOLLOW_SCORE, PREVIEWS_SIZE, diff --git a/server/middlewares/validators/logs.ts b/server/middlewares/validators/logs.ts index 07f3f552f..70e4d0d99 100644 --- a/server/middlewares/validators/logs.ts +++ b/server/middlewares/validators/logs.ts @@ -24,8 +24,25 @@ const getLogsValidator = [ } ] +const getAuditLogsValidator = [ + query('startDate') + .custom(isDateValid).withMessage('Should have a valid start date'), + query('endDate') + .optional() + .custom(isDateValid).withMessage('Should have a valid end date'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking getAuditLogsValidator parameters.', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + // --------------------------------------------------------------------------- export { - getLogsValidator + getLogsValidator, + getAuditLogsValidator } diff --git a/server/tests/api/server/logs.ts b/server/tests/api/server/logs.ts index 68f442199..d3c877408 100644 --- a/server/tests/api/server/logs.ts +++ b/server/tests/api/server/logs.ts @@ -2,17 +2,10 @@ import * as chai from 'chai' import 'mocha' -import { - flushTests, - killallServers, - flushAndRunServer, - ServerInfo, - setAccessTokensToServers, - cleanupTests -} from '../../../../shared/extra-utils/index' +import { cleanupTests, flushAndRunServer, ServerInfo, setAccessTokensToServers } from '../../../../shared/extra-utils/index' import { waitJobs } from '../../../../shared/extra-utils/server/jobs' import { uploadVideo } from '../../../../shared/extra-utils/videos/videos' -import { getLogs } from '../../../../shared/extra-utils/logs/logs' +import { getAuditLogs, getLogs } from '../../../../shared/extra-utils/logs/logs' const expect = chai.expect @@ -26,69 +19,123 @@ describe('Test logs', function () { await setAccessTokensToServers([ server ]) }) - it('Should get logs with a start date', async function () { - this.timeout(10000) + describe('With the standard log file', function () { + it('Should get logs with a start date', async function () { + this.timeout(10000) - await uploadVideo(server.url, server.accessToken, { name: 'video 1' }) - await waitJobs([ server ]) + await uploadVideo(server.url, server.accessToken, { name: 'video 1' }) + await waitJobs([ server ]) - const now = new Date() + const now = new Date() - await uploadVideo(server.url, server.accessToken, { name: 'video 2' }) - await waitJobs([ server ]) + await uploadVideo(server.url, server.accessToken, { name: 'video 2' }) + await waitJobs([ server ]) - const res = await getLogs(server.url, server.accessToken, now) - const logsString = JSON.stringify(res.body) + const res = await getLogs(server.url, server.accessToken, now) + const logsString = JSON.stringify(res.body) - expect(logsString.includes('video 1')).to.be.false - expect(logsString.includes('video 2')).to.be.true - }) + expect(logsString.includes('video 1')).to.be.false + expect(logsString.includes('video 2')).to.be.true + }) + + it('Should get logs with an end date', async function () { + this.timeout(20000) + + await uploadVideo(server.url, server.accessToken, { name: 'video 3' }) + await waitJobs([ server ]) + + const now1 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 4' }) + await waitJobs([ server ]) + + const now2 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 5' }) + await waitJobs([ server ]) + + const res = await getLogs(server.url, server.accessToken, now1, now2) + const logsString = JSON.stringify(res.body) - it('Should get logs with an end date', async function () { - this.timeout(20000) + expect(logsString.includes('video 3')).to.be.false + expect(logsString.includes('video 4')).to.be.true + expect(logsString.includes('video 5')).to.be.false + }) - await uploadVideo(server.url, server.accessToken, { name: 'video 3' }) - await waitJobs([ server ]) + it('Should get filter by level', async function () { + this.timeout(10000) - const now1 = new Date() + const now = new Date() - await uploadVideo(server.url, server.accessToken, { name: 'video 4' }) - await waitJobs([ server ]) + await uploadVideo(server.url, server.accessToken, { name: 'video 6' }) + await waitJobs([ server ]) - const now2 = new Date() + { + const res = await getLogs(server.url, server.accessToken, now, undefined, 'info') + const logsString = JSON.stringify(res.body) - await uploadVideo(server.url, server.accessToken, { name: 'video 5' }) - await waitJobs([ server ]) + expect(logsString.includes('video 6')).to.be.true + } - const res = await getLogs(server.url, server.accessToken, now1, now2) - const logsString = JSON.stringify(res.body) + { + const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn') + const logsString = JSON.stringify(res.body) - expect(logsString.includes('video 3')).to.be.false - expect(logsString.includes('video 4')).to.be.true - expect(logsString.includes('video 5')).to.be.false + expect(logsString.includes('video 6')).to.be.false + } + }) }) - it('Should get filter by level', async function () { - this.timeout(10000) + describe('With the audit log', function () { + it('Should get logs with a start date', async function () { + this.timeout(10000) - const now = new Date() + await uploadVideo(server.url, server.accessToken, { name: 'video 7' }) + await waitJobs([ server ]) - await uploadVideo(server.url, server.accessToken, { name: 'video 6' }) - await waitJobs([ server ]) + const now = new Date() - { - const res = await getLogs(server.url, server.accessToken, now, undefined, 'info') + await uploadVideo(server.url, server.accessToken, { name: 'video 8' }) + await waitJobs([ server ]) + + const res = await getAuditLogs(server.url, server.accessToken, now) const logsString = JSON.stringify(res.body) - expect(logsString.includes('video 6')).to.be.true - } + expect(logsString.includes('video 7')).to.be.false + expect(logsString.includes('video 8')).to.be.true + + expect(res.body).to.have.lengthOf(1) + + const item = res.body[0] + + const message = JSON.parse(item.message) + expect(message.domain).to.equal('videos') + expect(message.action).to.equal('create') + }) + + it('Should get logs with an end date', async function () { + this.timeout(20000) + + await uploadVideo(server.url, server.accessToken, { name: 'video 9' }) + await waitJobs([ server ]) + + const now1 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 10' }) + await waitJobs([ server ]) + + const now2 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 11' }) + await waitJobs([ server ]) - { - const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn') + const res = await getAuditLogs(server.url, server.accessToken, now1, now2) const logsString = JSON.stringify(res.body) - expect(logsString.includes('video 6')).to.be.false - } + expect(logsString.includes('video 9')).to.be.false + expect(logsString.includes('video 10')).to.be.true + expect(logsString.includes('video 11')).to.be.false + }) }) after(async function () { -- cgit v1.2.3