From 64553e8809271df1113e9143426a27f234410a74 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 20 Oct 2021 14:23:32 +0200 Subject: [PATCH] Add ability to filter logs by tags --- .../+admin/system/logs/logs.component.html | 2 + .../+admin/system/logs/logs.component.scss | 4 +- .../app/+admin/system/logs/logs.component.ts | 33 +++++++++------ .../app/+admin/system/logs/logs.service.ts | 9 ++-- .../select/select-tags.component.html | 2 +- .../select/select-tags.component.ts | 1 + server/controllers/api/server/logs.ts | 42 +++++++++++++++++-- server/middlewares/validators/logs.ts | 7 +++- server/tests/api/server/logs.ts | 23 +++++++++- shared/extra-utils/logs/logs-command.ts | 7 ++-- 10 files changed, 102 insertions(+), 28 deletions(-) diff --git a/client/src/app/+admin/system/logs/logs.component.html b/client/src/app/+admin/system/logs/logs.component.html index b2c7f84bc..18011b205 100644 --- a/client/src/app/+admin/system/logs/logs.component.html +++ b/client/src/app/+admin/system/logs/logs.component.html @@ -28,6 +28,8 @@ + + diff --git a/client/src/app/+admin/system/logs/logs.component.scss b/client/src/app/+admin/system/logs/logs.component.scss index fefa7efc2..be66d563b 100644 --- a/client/src/app/+admin/system/logs/logs.component.scss +++ b/client/src/app/+admin/system/logs/logs.component.scss @@ -52,9 +52,7 @@ @include peertube-select-container(150px); } - my-button, - .peertube-select-container, - ng-select { + > * { @include margin-left(10px); } } diff --git a/client/src/app/+admin/system/logs/logs.component.ts b/client/src/app/+admin/system/logs/logs.component.ts index 865ab80a2..06237522a 100644 --- a/client/src/app/+admin/system/logs/logs.component.ts +++ b/client/src/app/+admin/system/logs/logs.component.ts @@ -23,6 +23,7 @@ export class LogsComponent implements OnInit { startDate: string level: LogLevel logType: 'audit' | 'standard' + tagsOneOf: string[] = [] constructor ( private logsService: LogsService, @@ -51,20 +52,28 @@ export class LogsComponent implements OnInit { load () { this.loading = true - this.logsService.getLogs({ isAuditLog: this.isAuditLog(), level: this.level, startDate: this.startDate }) - .subscribe({ - next: logs => { - this.logs = logs - - setTimeout(() => { - this.logsElement.nativeElement.scrollIntoView({ block: 'end', inline: 'nearest' }) - }) - }, + const tagsOneOf = this.tagsOneOf.length !== 0 + ? this.tagsOneOf + : undefined + + this.logsService.getLogs({ + isAuditLog: this.isAuditLog(), + level: this.level, + startDate: this.startDate, + tagsOneOf + }).subscribe({ + next: logs => { + this.logs = logs + + setTimeout(() => { + this.logsElement.nativeElement.scrollIntoView({ block: 'end', inline: 'nearest' }) + }) + }, - error: err => this.notifier.error(err.message), + error: err => this.notifier.error(err.message), - complete: () => this.loading = false - }) + complete: () => this.loading = false + }) } isAuditLog () { diff --git a/client/src/app/+admin/system/logs/logs.service.ts b/client/src/app/+admin/system/logs/logs.service.ts index 0c222cad2..ea7e08b9b 100644 --- a/client/src/app/+admin/system/logs/logs.service.ts +++ b/client/src/app/+admin/system/logs/logs.service.ts @@ -2,7 +2,7 @@ import { Observable } from 'rxjs' import { catchError, map } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' -import { RestExtractor } from '@app/core' +import { RestExtractor, RestService } from '@app/core' import { LogLevel } from '@shared/models' import { environment } from '../../../../environments/environment' import { LogRow } from './log-row.model' @@ -14,22 +14,25 @@ export class LogsService { constructor ( private authHttp: HttpClient, + private restService: RestService, private restExtractor: RestExtractor ) {} getLogs (options: { isAuditLog: boolean startDate: string + tagsOneOf?: string[] level?: LogLevel endDate?: string }): Observable { - const { isAuditLog, startDate } = options + const { isAuditLog, startDate, endDate, tagsOneOf } = options let params = new HttpParams() params = params.append('startDate', startDate) if (!isAuditLog) params = params.append('level', options.level) - if (options.endDate) params.append('endDate', options.endDate) + if (endDate) params = params.append('endDate', options.endDate) + if (tagsOneOf) params = this.restService.addArrayParams(params, 'tagsOneOf', tagsOneOf) const path = isAuditLog ? LogsService.BASE_AUDIT_LOG_URL diff --git a/client/src/app/shared/shared-forms/select/select-tags.component.html b/client/src/app/shared/shared-forms/select/select-tags.component.html index e1cd50882..de6cee6db 100644 --- a/client/src/app/shared/shared-forms/select/select-tags.component.html +++ b/client/src/app/shared/shared-forms/select/select-tags.component.html @@ -2,7 +2,7 @@ [items]="availableItems" [(ngModel)]="selectedItems" (ngModelChange)="onModelChange()" - i18n-placeholder placeholder="Enter a new tag" + [placeholder]="placeholder" [maxSelectedItems]="5" [clearable]="true" [addTag]="true" diff --git a/client/src/app/shared/shared-forms/select/select-tags.component.ts b/client/src/app/shared/shared-forms/select/select-tags.component.ts index 93d199037..bef04de8a 100644 --- a/client/src/app/shared/shared-forms/select/select-tags.component.ts +++ b/client/src/app/shared/shared-forms/select/select-tags.component.ts @@ -16,6 +16,7 @@ import { NG_VALUE_ACCESSOR, ControlValueAccessor } from '@angular/forms' export class SelectTagsComponent implements ControlValueAccessor { @Input() availableItems: string[] = [] @Input() selectedItems: string[] = [] + @Input() placeholder = $localize`Enter a new tag` propagateChange = (_: any) => { /* empty */ } diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts index dfd5491aa..8aa4b7190 100644 --- a/server/controllers/api/server/logs.ts +++ b/server/controllers/api/server/logs.ts @@ -1,6 +1,7 @@ import express from 'express' import { readdir, readFile } from 'fs-extra' import { join } from 'path' +import { isArray } from '@server/helpers/custom-validators/misc' import { logger, mtimeSortFilesDesc } from '@server/helpers/logger' import { LogLevel } from '../../../../shared/models/server/log-level.type' import { UserRight } from '../../../../shared/models/users' @@ -51,20 +52,27 @@ async function getLogs (req: express.Request, res: express.Response) { startDateQuery: req.query.startDate, endDateQuery: req.query.endDate, level: req.query.level || 'info', + tagsOneOf: req.query.tagsOneOf, nameFilter: logNameFilter }) - return res.json(output).end() + return res.json(output) } async function generateOutput (options: { startDateQuery: string endDateQuery?: string + level: LogLevel nameFilter: RegExp + tagsOneOf?: string[] }) { const { startDateQuery, level, nameFilter } = options + const tagsOneOf = Array.isArray(options.tagsOneOf) && options.tagsOneOf.length !== 0 + ? new Set(options.tagsOneOf) + : undefined + const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR) let currentSize = 0 @@ -80,7 +88,7 @@ async function generateOutput (options: { const path = join(CONFIG.STORAGE.LOG_DIR, meta.file) logger.debug('Opening %s to fetch logs.', path) - const result = await getOutputFromFile(path, startDate, endDate, level, currentSize) + const result = await getOutputFromFile({ path, startDate, endDate, level, currentSize, tagsOneOf }) if (!result.output) break output = result.output.concat(output) @@ -92,9 +100,20 @@ async function generateOutput (options: { return output } -async function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) { +async function getOutputFromFile (options: { + path: string + startDate: Date + endDate: Date + level: LogLevel + currentSize: number + tagsOneOf: Set +}) { + const { path, startDate, endDate, level, tagsOneOf } = options + const startTime = startDate.getTime() const endTime = endDate.getTime() + let currentSize = options.currentSize + let logTime: number const logsLevel: { [ id in LogLevel ]: number } = { @@ -121,7 +140,12 @@ async function getOutputFromFile (path: string, startDate: Date, endDate: Date, } logTime = new Date(log.timestamp).getTime() - if (logTime >= startTime && logTime <= endTime && logsLevel[log.level] >= logsLevel[level]) { + if ( + logTime >= startTime && + logTime <= endTime && + logsLevel[log.level] >= logsLevel[level] && + (!tagsOneOf || lineHasTag(log, tagsOneOf)) + ) { output.push(log) currentSize += line.length @@ -135,6 +159,16 @@ async function getOutputFromFile (path: string, startDate: Date, endDate: Date, return { currentSize, output: output.reverse(), logTime } } +function lineHasTag (line: { tags?: string }, tagsOneOf: Set) { + if (!isArray(line.tags)) return false + + for (const lineTag of line.tags) { + if (tagsOneOf.has(lineTag)) return true + } + + return false +} + function generateLogNameFilter (baseName: string) { return new RegExp('^' + baseName.replace(/\.log$/, '') + '\\d*.log$') } diff --git a/server/middlewares/validators/logs.ts b/server/middlewares/validators/logs.ts index 03c1c4df1..901d8ca64 100644 --- a/server/middlewares/validators/logs.ts +++ b/server/middlewares/validators/logs.ts @@ -1,7 +1,8 @@ import express from 'express' import { query } from 'express-validator' +import { isStringArray } from '@server/helpers/custom-validators/search' import { isValidLogLevel } from '../../helpers/custom-validators/logs' -import { isDateValid } from '../../helpers/custom-validators/misc' +import { isDateValid, toArray } from '../../helpers/custom-validators/misc' import { logger } from '../../helpers/logger' import { areValidationErrors } from './shared' @@ -11,6 +12,10 @@ const getLogsValidator = [ query('level') .optional() .custom(isValidLogLevel).withMessage('Should have a valid level'), + query('tagsOneOf') + .optional() + .customSanitizer(toArray) + .custom(isStringArray).withMessage('Should have a valid one of tags array'), query('endDate') .optional() .custom(isDateValid).withMessage('Should have an end date that conforms to ISO 8601'), diff --git a/server/tests/api/server/logs.ts b/server/tests/api/server/logs.ts index bcd94dda3..4fa13886e 100644 --- a/server/tests/api/server/logs.ts +++ b/server/tests/api/server/logs.ts @@ -71,7 +71,7 @@ describe('Test logs', function () { expect(logsString.includes('video 5')).to.be.false }) - it('Should get filter by level', async function () { + it('Should filter by level', async function () { this.timeout(20000) const now = new Date() @@ -94,6 +94,27 @@ describe('Test logs', function () { } }) + it('Should filter by tag', async function () { + const now = new Date() + + const { uuid } = await server.videos.upload({ attributes: { name: 'video 6' } }) + await waitJobs([ server ]) + + { + const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ 'toto' ] }) + expect(body).to.have.lengthOf(0) + } + + { + const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ uuid ] }) + expect(body).to.not.have.lengthOf(0) + + for (const line of body) { + expect(line.tags).to.contain(uuid) + } + } + }) + it('Should log ping requests', async function () { this.timeout(10000) diff --git a/shared/extra-utils/logs/logs-command.ts b/shared/extra-utils/logs/logs-command.ts index 5912e814f..7b5c66c0c 100644 --- a/shared/extra-utils/logs/logs-command.ts +++ b/shared/extra-utils/logs/logs-command.ts @@ -8,15 +8,16 @@ export class LogsCommand extends AbstractCommand { startDate: Date endDate?: Date level?: LogLevel + tagsOneOf?: string[] }) { - const { startDate, endDate, level } = options + const { startDate, endDate, tagsOneOf, level } = options const path = '/api/v1/server/logs' - return this.getRequestBody({ + return this.getRequestBody({ ...options, path, - query: { startDate, endDate, level }, + query: { startDate, endDate, level, tagsOneOf }, implicitToken: true, defaultExpectedStatus: HttpStatusCode.OK_200 }) -- 2.41.0