1 import express from 'express'
2 import { readdir, readFile } from 'fs-extra'
3 import { join } from 'path'
4 import { isArray } from '@server/helpers/custom-validators/misc'
5 import { logger, mtimeSortFilesDesc } from '@server/helpers/logger'
6 import { pick } from '@shared/core-utils'
7 import { ClientLogCreate, HttpStatusCode } from '@shared/models'
8 import { ServerLogLevel } from '../../../../shared/models/server/server-log-level.type'
9 import { UserRight } from '../../../../shared/models/users'
10 import { CONFIG } from '../../../initializers/config'
11 import { AUDIT_LOG_FILENAME, LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants'
12 import { asyncMiddleware, authenticate, buildRateLimiter, ensureUserHasRight, optionalAuthenticate } from '../../../middlewares'
13 import { createClientLogValidator, getAuditLogsValidator, getLogsValidator } from '../../../middlewares/validators/logs'
15 const createClientLogRateLimiter = buildRateLimiter({
16 windowMs: CONFIG.RATES_LIMIT.RECEIVE_CLIENT_LOG.WINDOW_MS,
17 max: CONFIG.RATES_LIMIT.RECEIVE_CLIENT_LOG.MAX
20 const logsRouter = express.Router()
22 logsRouter.post('/logs/client',
23 createClientLogRateLimiter,
25 createClientLogValidator,
29 logsRouter.get('/logs',
31 ensureUserHasRight(UserRight.MANAGE_LOGS),
33 asyncMiddleware(getLogs)
36 logsRouter.get('/audit-logs',
38 ensureUserHasRight(UserRight.MANAGE_LOGS),
39 getAuditLogsValidator,
40 asyncMiddleware(getAuditLogs)
43 // ---------------------------------------------------------------------------
49 // ---------------------------------------------------------------------------
51 function createClientLog (req: express.Request, res: express.Response) {
52 const logInfo = req.body as ClientLogCreate
56 username: res.locals.oauth?.token?.User?.username,
58 ...pick(logInfo, [ 'userAgent', 'stackTrace', 'meta', 'url' ])
61 logger.log(logInfo.level, `Client log: ${logInfo.message}`, meta)
63 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
66 const auditLogNameFilter = generateLogNameFilter(AUDIT_LOG_FILENAME)
67 async function getAuditLogs (req: express.Request, res: express.Response) {
68 const output = await generateOutput({
69 startDateQuery: req.query.startDate,
70 endDateQuery: req.query.endDate,
72 nameFilter: auditLogNameFilter
75 return res.json(output).end()
78 const logNameFilter = generateLogNameFilter(LOG_FILENAME)
79 async function getLogs (req: express.Request, res: express.Response) {
80 const output = await generateOutput({
81 startDateQuery: req.query.startDate,
82 endDateQuery: req.query.endDate,
83 level: req.query.level || 'info',
84 tagsOneOf: req.query.tagsOneOf,
85 nameFilter: logNameFilter
88 return res.json(output)
91 async function generateOutput (options: {
92 startDateQuery: string
99 const { startDateQuery, level, nameFilter } = options
101 const tagsOneOf = Array.isArray(options.tagsOneOf) && options.tagsOneOf.length !== 0
102 ? new Set(options.tagsOneOf)
105 const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
106 const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR)
109 const startDate = new Date(startDateQuery)
110 const endDate = options.endDateQuery ? new Date(options.endDateQuery) : new Date()
112 let output: string[] = []
114 for (const meta of sortedLogFiles) {
115 if (nameFilter.exec(meta.file) === null) continue
117 const path = join(CONFIG.STORAGE.LOG_DIR, meta.file)
118 logger.debug('Opening %s to fetch logs.', path)
120 const result = await getOutputFromFile({ path, startDate, endDate, level, currentSize, tagsOneOf })
121 if (!result.output) break
123 output = result.output.concat(output)
124 currentSize = result.currentSize
126 if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS || (result.logTime && result.logTime < startDate.getTime())) break
132 async function getOutputFromFile (options: {
136 level: ServerLogLevel
138 tagsOneOf: Set<string>
140 const { path, startDate, endDate, level, tagsOneOf } = options
142 const startTime = startDate.getTime()
143 const endTime = endDate.getTime()
144 let currentSize = options.currentSize
148 const logsLevel: { [ id in ServerLogLevel ]: number } = {
156 const content = await readFile(path)
157 const lines = content.toString().split('\n')
158 const output: any[] = []
160 for (let i = lines.length - 1; i >= 0; i--) {
161 const line = lines[i]
165 log = JSON.parse(line)
167 // Maybe there a multiple \n at the end of the file
171 logTime = new Date(log.timestamp).getTime()
173 logTime >= startTime &&
174 logTime <= endTime &&
175 logsLevel[log.level] >= logsLevel[level] &&
176 (!tagsOneOf || lineHasTag(log, tagsOneOf))
180 currentSize += line.length
182 if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) break
183 } else if (logTime < startTime) {
188 return { currentSize, output: output.reverse(), logTime }
191 function lineHasTag (line: { tags?: string }, tagsOneOf: Set<string>) {
192 if (!isArray(line.tags)) return false
194 for (const lineTag of line.tags) {
195 if (tagsOneOf.has(lineTag)) return true
201 function generateLogNameFilter (baseName: string) {
202 return new RegExp('^' + baseName.replace(/\.log$/, '') + '\\d*.log$')