From fd8710b897a67518d3a61c319e54b6a65ba443ef Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 10 Apr 2019 15:26:33 +0200 Subject: Add logs endpoint --- client/src/app/+admin/jobs/index.ts | 4 - client/src/app/+admin/jobs/job.component.ts | 6 -- client/src/app/+admin/jobs/job.routes.ts | 32 ------ client/src/app/+admin/jobs/jobs-list/index.ts | 1 - .../+admin/jobs/jobs-list/jobs-list.component.html | 56 ---------- .../+admin/jobs/jobs-list/jobs-list.component.scss | 14 --- .../+admin/jobs/jobs-list/jobs-list.component.ts | 69 ------------ client/src/app/+admin/jobs/shared/index.ts | 1 - client/src/app/+admin/jobs/shared/job.service.ts | 46 -------- client/src/app/+admin/system/jobs/index.ts | 4 + client/src/app/+admin/system/jobs/job.service.ts | 46 ++++++++ .../src/app/+admin/system/jobs/jobs.component.html | 56 ++++++++++ .../src/app/+admin/system/jobs/jobs.component.scss | 14 +++ .../src/app/+admin/system/jobs/jobs.component.ts | 69 ++++++++++++ scripts/parse-log.ts | 58 +++++----- server/controllers/api/server/index.ts | 2 + server/controllers/api/server/logs.ts | 90 ++++++++++++++++ server/helpers/custom-validators/logs.ts | 14 +++ server/helpers/logger.ts | 11 +- server/initializers/constants.ts | 3 + server/middlewares/validators/logs.ts | 31 ++++++ server/middlewares/validators/videos/videos.ts | 8 +- server/tests/api/check-params/index.ts | 1 + server/tests/api/check-params/logs.ts | 117 +++++++++++++++++++++ server/tests/api/server/index.ts | 1 + server/tests/api/server/logs.ts | 92 ++++++++++++++++ shared/models/server/log-level.type.ts | 1 + shared/models/users/user-right.enum.ts | 2 + shared/utils/logs/logs.ts | 41 ++++++++ 29 files changed, 624 insertions(+), 266 deletions(-) delete mode 100644 client/src/app/+admin/jobs/index.ts delete mode 100644 client/src/app/+admin/jobs/job.component.ts delete mode 100644 client/src/app/+admin/jobs/job.routes.ts delete mode 100644 client/src/app/+admin/jobs/jobs-list/index.ts delete mode 100644 client/src/app/+admin/jobs/jobs-list/jobs-list.component.html delete mode 100644 client/src/app/+admin/jobs/jobs-list/jobs-list.component.scss delete mode 100644 client/src/app/+admin/jobs/jobs-list/jobs-list.component.ts delete mode 100644 client/src/app/+admin/jobs/shared/index.ts delete mode 100644 client/src/app/+admin/jobs/shared/job.service.ts create mode 100644 client/src/app/+admin/system/jobs/index.ts create mode 100644 client/src/app/+admin/system/jobs/job.service.ts create mode 100644 client/src/app/+admin/system/jobs/jobs.component.html create mode 100644 client/src/app/+admin/system/jobs/jobs.component.scss create mode 100644 client/src/app/+admin/system/jobs/jobs.component.ts create mode 100644 server/controllers/api/server/logs.ts create mode 100644 server/helpers/custom-validators/logs.ts create mode 100644 server/middlewares/validators/logs.ts create mode 100644 server/tests/api/check-params/logs.ts create mode 100644 server/tests/api/server/logs.ts create mode 100644 shared/models/server/log-level.type.ts create mode 100644 shared/utils/logs/logs.ts diff --git a/client/src/app/+admin/jobs/index.ts b/client/src/app/+admin/jobs/index.ts deleted file mode 100644 index c0e0cc95d..000000000 --- a/client/src/app/+admin/jobs/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './shared' -export * from './jobs-list' -export * from './job.routes' -export * from './job.component' diff --git a/client/src/app/+admin/jobs/job.component.ts b/client/src/app/+admin/jobs/job.component.ts deleted file mode 100644 index bc80c9a6a..000000000 --- a/client/src/app/+admin/jobs/job.component.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Component } from '@angular/core' - -@Component({ - template: '' -}) -export class JobsComponent {} diff --git a/client/src/app/+admin/jobs/job.routes.ts b/client/src/app/+admin/jobs/job.routes.ts deleted file mode 100644 index 331dc2af2..000000000 --- a/client/src/app/+admin/jobs/job.routes.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Routes } from '@angular/router' -import { UserRight } from '../../../../../shared' -import { UserRightGuard } from '../../core' -import { JobsComponent } from './job.component' -import { JobsListComponent } from './jobs-list/jobs-list.component' - -export const JobsRoutes: Routes = [ - { - path: 'jobs', - component: JobsComponent, - canActivate: [ UserRightGuard ], - data: { - userRight: UserRight.MANAGE_JOBS - }, - children: [ - { - path: '', - redirectTo: 'list', - pathMatch: 'full' - }, - { - path: 'list', - component: JobsListComponent, - data: { - meta: { - title: 'Jobs list' - } - } - } - ] - } -] diff --git a/client/src/app/+admin/jobs/jobs-list/index.ts b/client/src/app/+admin/jobs/jobs-list/index.ts deleted file mode 100644 index cf590a6f8..000000000 --- a/client/src/app/+admin/jobs/jobs-list/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './jobs-list.component' diff --git a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.html b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.html deleted file mode 100644 index 7ed1888e2..000000000 --- a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.html +++ /dev/null @@ -1,56 +0,0 @@ -
-
Jobs list
- -
- -
-
- - - - - - ID - Type - State - Created - Processed on - Finished on - - - - - - - - - - - {{ job.id }} - {{ job.type }} - {{ job.state }} - {{ job.createdAt }} - {{ job.processedOn }} - {{ job.finishedOn }} - - - - - - - 
{{ job.data }}
- - - - - 
{{ job.error }}
- - - - - diff --git a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.scss b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.scss deleted file mode 100644 index ab05f1982..000000000 --- a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.scss +++ /dev/null @@ -1,14 +0,0 @@ -@import '_variables'; -@import '_mixins'; - -.peertube-select-container { - @include peertube-select-container(auto); -} - -pre { - font-size: 11px; -} - -.job-error { - color: red; -} diff --git a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.ts b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.ts deleted file mode 100644 index b265e1dd6..000000000 --- a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Component, OnInit } from '@angular/core' -import { peertubeLocalStorage } from '@app/shared/misc/peertube-local-storage' -import { Notifier } from '@app/core' -import { SortMeta } from 'primeng/primeng' -import { Job } from '../../../../../../shared/index' -import { JobState } from '../../../../../../shared/models' -import { RestPagination, RestTable } from '../../../shared' -import { JobService } from '../shared' -import { I18n } from '@ngx-translate/i18n-polyfill' - -@Component({ - selector: 'my-jobs-list', - templateUrl: './jobs-list.component.html', - styleUrls: [ './jobs-list.component.scss' ] -}) -export class JobsListComponent extends RestTable implements OnInit { - private static JOB_STATE_LOCAL_STORAGE_STATE = 'jobs-list-state' - - jobState: JobState = 'waiting' - jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] - jobs: Job[] = [] - totalRecords: number - rowsPerPage = 10 - sort: SortMeta = { field: 'createdAt', order: -1 } - pagination: RestPagination = { count: this.rowsPerPage, start: 0 } - - constructor ( - private notifier: Notifier, - private jobsService: JobService, - private i18n: I18n - ) { - super() - } - - ngOnInit () { - this.loadJobState() - this.initialize() - } - - onJobStateChanged () { - this.pagination.start = 0 - - this.loadData() - this.saveJobState() - } - - protected loadData () { - this.jobsService - .getJobs(this.jobState, this.pagination, this.sort) - .subscribe( - resultList => { - this.jobs = resultList.data - this.totalRecords = resultList.total - }, - - err => this.notifier.error(err.message) - ) - } - - private loadJobState () { - const result = peertubeLocalStorage.getItem(JobsListComponent.JOB_STATE_LOCAL_STORAGE_STATE) - - if (result) this.jobState = result as JobState - } - - private saveJobState () { - peertubeLocalStorage.setItem(JobsListComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState) - } -} diff --git a/client/src/app/+admin/jobs/shared/index.ts b/client/src/app/+admin/jobs/shared/index.ts deleted file mode 100644 index 609439e5c..000000000 --- a/client/src/app/+admin/jobs/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './job.service' diff --git a/client/src/app/+admin/jobs/shared/job.service.ts b/client/src/app/+admin/jobs/shared/job.service.ts deleted file mode 100644 index b96dc3359..000000000 --- a/client/src/app/+admin/jobs/shared/job.service.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { catchError, map } from 'rxjs/operators' -import { HttpClient, HttpParams } from '@angular/common/http' -import { Injectable } from '@angular/core' -import { SortMeta } from 'primeng/primeng' -import { Observable } from 'rxjs' -import { ResultList } from '../../../../../../shared' -import { JobState } from '../../../../../../shared/models' -import { Job } from '../../../../../../shared/models/server/job.model' -import { environment } from '../../../../environments/environment' -import { RestExtractor, RestPagination, RestService } from '../../../shared' - -@Injectable() -export class JobService { - private static BASE_JOB_URL = environment.apiUrl + '/api/v1/jobs' - - constructor ( - private authHttp: HttpClient, - private restService: RestService, - private restExtractor: RestExtractor - ) {} - - getJobs (state: JobState, pagination: RestPagination, sort: SortMeta): Observable> { - let params = new HttpParams() - params = this.restService.addRestGetParams(params, pagination, sort) - - return this.authHttp.get>(JobService.BASE_JOB_URL + '/' + state, { params }) - .pipe( - map(res => { - return this.restExtractor.convertResultListDateToHuman(res, [ 'createdAt', 'processedOn', 'finishedOn' ]) - }), - map(res => this.restExtractor.applyToResultListData(res, this.prettyPrintData)), - map(res => this.restExtractor.applyToResultListData(res, this.buildUniqId)), - catchError(err => this.restExtractor.handleError(err)) - ) - } - - private prettyPrintData (obj: Job) { - const data = JSON.stringify(obj.data, null, 2) - - return Object.assign(obj, { data }) - } - - private buildUniqId (obj: Job) { - return Object.assign(obj, { uniqId: `${obj.id}-${obj.type}` }) - } -} diff --git a/client/src/app/+admin/system/jobs/index.ts b/client/src/app/+admin/system/jobs/index.ts new file mode 100644 index 000000000..c0e0cc95d --- /dev/null +++ b/client/src/app/+admin/system/jobs/index.ts @@ -0,0 +1,4 @@ +export * from './shared' +export * from './jobs-list' +export * from './job.routes' +export * from './job.component' diff --git a/client/src/app/+admin/system/jobs/job.service.ts b/client/src/app/+admin/system/jobs/job.service.ts new file mode 100644 index 000000000..b96dc3359 --- /dev/null +++ b/client/src/app/+admin/system/jobs/job.service.ts @@ -0,0 +1,46 @@ +import { catchError, map } from 'rxjs/operators' +import { HttpClient, HttpParams } from '@angular/common/http' +import { Injectable } from '@angular/core' +import { SortMeta } from 'primeng/primeng' +import { Observable } from 'rxjs' +import { ResultList } from '../../../../../../shared' +import { JobState } from '../../../../../../shared/models' +import { Job } from '../../../../../../shared/models/server/job.model' +import { environment } from '../../../../environments/environment' +import { RestExtractor, RestPagination, RestService } from '../../../shared' + +@Injectable() +export class JobService { + private static BASE_JOB_URL = environment.apiUrl + '/api/v1/jobs' + + constructor ( + private authHttp: HttpClient, + private restService: RestService, + private restExtractor: RestExtractor + ) {} + + getJobs (state: JobState, pagination: RestPagination, sort: SortMeta): Observable> { + let params = new HttpParams() + params = this.restService.addRestGetParams(params, pagination, sort) + + return this.authHttp.get>(JobService.BASE_JOB_URL + '/' + state, { params }) + .pipe( + map(res => { + return this.restExtractor.convertResultListDateToHuman(res, [ 'createdAt', 'processedOn', 'finishedOn' ]) + }), + map(res => this.restExtractor.applyToResultListData(res, this.prettyPrintData)), + map(res => this.restExtractor.applyToResultListData(res, this.buildUniqId)), + catchError(err => this.restExtractor.handleError(err)) + ) + } + + private prettyPrintData (obj: Job) { + const data = JSON.stringify(obj.data, null, 2) + + return Object.assign(obj, { data }) + } + + private buildUniqId (obj: Job) { + return Object.assign(obj, { uniqId: `${obj.id}-${obj.type}` }) + } +} diff --git a/client/src/app/+admin/system/jobs/jobs.component.html b/client/src/app/+admin/system/jobs/jobs.component.html new file mode 100644 index 000000000..7ed1888e2 --- /dev/null +++ b/client/src/app/+admin/system/jobs/jobs.component.html @@ -0,0 +1,56 @@ +
+
Jobs list
+ +
+ +
+
+ + + + + + ID + Type + State + Created + Processed on + Finished on + + + + + + + + + + + {{ job.id }} + {{ job.type }} + {{ job.state }} + {{ job.createdAt }} + {{ job.processedOn }} + {{ job.finishedOn }} + + + + + + + 
{{ job.data }}
+ + + + + 
{{ job.error }}
+ + + + + diff --git a/client/src/app/+admin/system/jobs/jobs.component.scss b/client/src/app/+admin/system/jobs/jobs.component.scss new file mode 100644 index 000000000..ab05f1982 --- /dev/null +++ b/client/src/app/+admin/system/jobs/jobs.component.scss @@ -0,0 +1,14 @@ +@import '_variables'; +@import '_mixins'; + +.peertube-select-container { + @include peertube-select-container(auto); +} + +pre { + font-size: 11px; +} + +.job-error { + color: red; +} diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts new file mode 100644 index 000000000..b265e1dd6 --- /dev/null +++ b/client/src/app/+admin/system/jobs/jobs.component.ts @@ -0,0 +1,69 @@ +import { Component, OnInit } from '@angular/core' +import { peertubeLocalStorage } from '@app/shared/misc/peertube-local-storage' +import { Notifier } from '@app/core' +import { SortMeta } from 'primeng/primeng' +import { Job } from '../../../../../../shared/index' +import { JobState } from '../../../../../../shared/models' +import { RestPagination, RestTable } from '../../../shared' +import { JobService } from '../shared' +import { I18n } from '@ngx-translate/i18n-polyfill' + +@Component({ + selector: 'my-jobs-list', + templateUrl: './jobs-list.component.html', + styleUrls: [ './jobs-list.component.scss' ] +}) +export class JobsListComponent extends RestTable implements OnInit { + private static JOB_STATE_LOCAL_STORAGE_STATE = 'jobs-list-state' + + jobState: JobState = 'waiting' + jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] + jobs: Job[] = [] + totalRecords: number + rowsPerPage = 10 + sort: SortMeta = { field: 'createdAt', order: -1 } + pagination: RestPagination = { count: this.rowsPerPage, start: 0 } + + constructor ( + private notifier: Notifier, + private jobsService: JobService, + private i18n: I18n + ) { + super() + } + + ngOnInit () { + this.loadJobState() + this.initialize() + } + + onJobStateChanged () { + this.pagination.start = 0 + + this.loadData() + this.saveJobState() + } + + protected loadData () { + this.jobsService + .getJobs(this.jobState, this.pagination, this.sort) + .subscribe( + resultList => { + this.jobs = resultList.data + this.totalRecords = resultList.total + }, + + err => this.notifier.error(err.message) + ) + } + + private loadJobState () { + const result = peertubeLocalStorage.getItem(JobsListComponent.JOB_STATE_LOCAL_STORAGE_STATE) + + if (result) this.jobState = result as JobState + } + + private saveJobState () { + peertubeLocalStorage.setItem(JobsListComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState) + } +} diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts index 86aaa7994..66a5b8719 100755 --- a/scripts/parse-log.ts +++ b/scripts/parse-log.ts @@ -1,10 +1,11 @@ import * as program from 'commander' -import { createReadStream, readdirSync, statSync } from 'fs-extra' +import { createReadStream, readdir } from 'fs-extra' import { join } from 'path' import { createInterface } from 'readline' import * as winston from 'winston' import { labelFormatter } from '../server/helpers/logger' import { CONFIG } from '../server/initializers/constants' +import { mtimeSortFilesDesc } from '../shared/utils/logs/logs' program .option('-l, --level [level]', 'Level log (debug/info/warn/error)') @@ -52,42 +53,47 @@ const logLevels = { debug: logger.debug.bind(logger) } -const logFiles = readdirSync(CONFIG.STORAGE.LOG_DIR) -const lastLogFile = getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR) +run() + .then(() => process.exit(0)) + .catch(err => console.error(err)) -const path = join(CONFIG.STORAGE.LOG_DIR, lastLogFile) -console.log('Opening %s.', path) +function run () { + return new Promise(async res => { + const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) + const lastLogFile = await getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR) -const rl = createInterface({ - input: createReadStream(path) -}) + const path = join(CONFIG.STORAGE.LOG_DIR, lastLogFile) + console.log('Opening %s.', path) -rl.on('line', line => { - const log = JSON.parse(line) - // Don't know why but loggerFormat does not remove splat key - Object.assign(log, { splat: undefined }) + const stream = createReadStream(path) - logLevels[log.level](log) -}) + const rl = createInterface({ + input: stream + }) -function toTimeFormat (time: string) { - const timestamp = Date.parse(time) + rl.on('line', line => { + const log = JSON.parse(line) + // Don't know why but loggerFormat does not remove splat key + Object.assign(log, { splat: undefined }) - if (isNaN(timestamp) === true) return 'Unknown date' + logLevels[ log.level ](log) + }) - return new Date(timestamp).toISOString() + stream.once('close', () => res()) + }) } // Thanks: https://stackoverflow.com/a/37014317 -function getNewestFile (files: string[], basePath: string) { - const out = [] +async function getNewestFile (files: string[], basePath: string) { + const sorted = await mtimeSortFilesDesc(files, basePath) - files.forEach(file => { - const stats = statSync(basePath + '/' + file) - if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() }) - }) + return (sorted.length > 0) ? sorted[ 0 ].file : '' +} + +function toTimeFormat (time: string) { + const timestamp = Date.parse(time) - out.sort((a, b) => b.mtime - a.mtime) + if (isNaN(timestamp) === true) return 'Unknown date' - return (out.length > 0) ? out[ 0 ].file : '' + return new Date(timestamp).toISOString() } diff --git a/server/controllers/api/server/index.ts b/server/controllers/api/server/index.ts index 814248e5f..de09588df 100644 --- a/server/controllers/api/server/index.ts +++ b/server/controllers/api/server/index.ts @@ -4,6 +4,7 @@ import { statsRouter } from './stats' import { serverRedundancyRouter } from './redundancy' import { serverBlocklistRouter } from './server-blocklist' import { contactRouter } from './contact' +import { logsRouter } from './logs' const serverRouter = express.Router() @@ -12,6 +13,7 @@ serverRouter.use('/', serverRedundancyRouter) serverRouter.use('/', statsRouter) serverRouter.use('/', serverBlocklistRouter) serverRouter.use('/', contactRouter) +serverRouter.use('/', logsRouter) // --------------------------------------------------------------------------- diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts new file mode 100644 index 000000000..c551c67e3 --- /dev/null +++ b/server/controllers/api/server/logs.ts @@ -0,0 +1,90 @@ +import * as express from 'express' +import { UserRight } from '../../../../shared/models/users' +import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' +import { mtimeSortFilesDesc } from '../../../../shared/utils/logs/logs' +import { readdir } from 'fs-extra' +import { CONFIG, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers' +import { createInterface } from 'readline' +import { createReadStream } from 'fs' +import { join } from 'path' +import { getLogsValidator } from '../../../middlewares/validators/logs' +import { LogLevel } from '../../../../shared/models/server/log-level.type' + +const logsRouter = express.Router() + +logsRouter.get('/logs', + authenticate, + ensureUserHasRight(UserRight.MANAGE_LOGS), + getLogsValidator, + asyncMiddleware(getLogs) +) + +// --------------------------------------------------------------------------- + +export { + logsRouter +} + +// --------------------------------------------------------------------------- + +async function getLogs (req: express.Request, res: express.Response) { + const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) + const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR) + let currentSize = 0 + + const startDate = new Date(req.query.startDate) + const endDate = req.query.endDate ? new Date(req.query.endDate) : new Date() + const level: LogLevel = req.query.level || 'info' + + let output = '' + + for (const meta of sortedLogFiles) { + const path = join(CONFIG.STORAGE.LOG_DIR, meta.file) + + const result = await getOutputFromFile(path, startDate, endDate, level, currentSize) + if (!result.output) break + + output = output + result.output + currentSize = result.currentSize + + if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) break + } + + return res.json(output).end() +} + +function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) { + const startTime = startDate.getTime() + const endTime = endDate.getTime() + + const logsLevel: { [ id in LogLevel ]: number } = { + debug: 0, + info: 1, + warn: 2, + error: 3 + } + + return new Promise<{ output: string, currentSize: number }>(res => { + const stream = createReadStream(path) + let output = '' + + stream.once('close', () => res({ output, currentSize })) + + const rl = createInterface({ + input: stream + }) + + rl.on('line', line => { + const log = JSON.parse(line) + + const logTime = new Date(log.timestamp).getTime() + if (logTime >= startTime && logTime <= endTime && logsLevel[log.level] >= logsLevel[level]) { + output += line + + currentSize += line.length + + if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) stream.close() + } + }) + }) +} diff --git a/server/helpers/custom-validators/logs.ts b/server/helpers/custom-validators/logs.ts new file mode 100644 index 000000000..30d0ce262 --- /dev/null +++ b/server/helpers/custom-validators/logs.ts @@ -0,0 +1,14 @@ +import { exists } from './misc' +import { LogLevel } from '../../../shared/models/server/log-level.type' + +const logLevels: LogLevel[] = [ 'debug', 'info', 'warn', 'error' ] + +function isValidLogLevel (value: any) { + return exists(value) && logLevels.indexOf(value) !== -1 +} + +// --------------------------------------------------------------------------- + +export { + isValidLogLevel +} diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 203e637a8..f8a142718 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -3,10 +3,12 @@ import { mkdirpSync } from 'fs-extra' import * as path from 'path' import * as winston from 'winston' import { CONFIG } from '../initializers' +import { omit } from 'lodash' const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT // Create the directory if it does not exist +// FIXME: use async mkdirpSync(CONFIG.STORAGE.LOG_DIR) function loggerReplacer (key: string, value: any) { @@ -22,13 +24,10 @@ function loggerReplacer (key: string, value: any) { } const consoleLoggerFormat = winston.format.printf(info => { - const obj = { - meta: info.meta, - err: info.err, - sql: info.sql - } + const obj = omit(info, 'label', 'timestamp', 'level', 'message') let additionalInfos = JSON.stringify(obj, loggerReplacer, 2) + if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' else additionalInfos = ' ' + additionalInfos @@ -57,7 +56,7 @@ const logger = winston.createLogger({ filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube.log'), handleExceptions: true, maxsize: 1024 * 1024 * 12, - maxFiles: 5, + maxFiles: 20, format: winston.format.combine( winston.format.timestamp(), jsonLoggerFormat diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 3f02572db..739ea5502 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -730,6 +730,8 @@ const FEEDS = { COUNT: 20 } +const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000 + // --------------------------------------------------------------------------- const TRACKER_RATE_LIMITS = { @@ -819,6 +821,7 @@ export { STATIC_PATHS, VIDEO_IMPORT_TIMEOUT, VIDEO_PLAYLIST_TYPES, + MAX_LOGS_OUTPUT_CHARACTERS, ACTIVITY_PUB, ACTIVITY_PUB_ACTOR_TYPES, THUMBNAILS_SIZE, diff --git a/server/middlewares/validators/logs.ts b/server/middlewares/validators/logs.ts new file mode 100644 index 000000000..7380c6edd --- /dev/null +++ b/server/middlewares/validators/logs.ts @@ -0,0 +1,31 @@ +import * as express from 'express' +import { logger } from '../../helpers/logger' +import { areValidationErrors } from './utils' +import { isDateValid } from '../../helpers/custom-validators/misc' +import { query } from 'express-validator/check' +import { isValidLogLevel } from '../../helpers/custom-validators/logs' + +const getLogsValidator = [ + query('startDate') + .custom(isDateValid).withMessage('Should have a valid start date'), + query('level') + .optional() + .custom(isValidLogLevel).withMessage('Should have a valid level'), + query('endDate') + .optional() + .custom(isDateValid).withMessage('Should have a valid end date'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking getLogsValidator parameters.', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + getLogsValidator +} diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index b70abf429..e247db708 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -14,18 +14,18 @@ import { } from '../../../helpers/custom-validators/misc' import { checkUserCanManageVideo, - isVideoOriginallyPublishedAtValid, + doesVideoChannelOfAccountExist, + doesVideoExist, isScheduleVideoUpdatePrivacyValid, isVideoCategoryValid, - doesVideoChannelOfAccountExist, isVideoDescriptionValid, - doesVideoExist, isVideoFile, isVideoFilterValid, isVideoImage, isVideoLanguageValid, isVideoLicenceValid, isVideoNameValid, + isVideoOriginallyPublishedAtValid, isVideoPrivacyValid, isVideoSupportValid, isVideoTagsValid @@ -37,10 +37,8 @@ import { authenticatePromiseIfNeeded } from '../../oauth' import { areValidationErrors } from '../utils' import { cleanUpReqFiles } from '../../../helpers/express-utils' import { VideoModel } from '../../../models/video/video' -import { UserModel } from '../../../models/account/user' import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } from '../../../helpers/custom-validators/video-ownership' import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model' -import { VideoChangeOwnershipModel } from '../../../models/video/video-change-ownership' import { AccountModel } from '../../../models/account/account' import { VideoFetchType } from '../../../helpers/video' import { isNSFWQueryValid, isNumberArray, isStringArray } from '../../../helpers/custom-validators/search' diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index ca51cd39a..bdac95025 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -4,6 +4,7 @@ import './config' import './contact-form' import './follows' import './jobs' +import './logs' import './redundancy' import './search' import './services' diff --git a/server/tests/api/check-params/logs.ts b/server/tests/api/check-params/logs.ts new file mode 100644 index 000000000..d6a40da61 --- /dev/null +++ b/server/tests/api/check-params/logs.ts @@ -0,0 +1,117 @@ +/* tslint:disable:no-unused-expression */ + +import 'mocha' + +import { + createUser, + flushTests, + killallServers, + runServer, + ServerInfo, + setAccessTokensToServers, + userLogin +} from '../../../../shared/utils' +import { makeGetRequest } from '../../../../shared/utils/requests/requests' + +describe('Test logs API validators', function () { + const path = '/api/v1/server/logs' + let server: ServerInfo + let userAccessToken = '' + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + await flushTests() + + server = await runServer(1) + + await setAccessTokensToServers([ server ]) + + const user = { + username: 'user1', + password: 'my super password' + } + await createUser(server.url, server.accessToken, user.username, user.password) + userAccessToken = await userLogin(server, user) + }) + + describe('When getting logs', function () { + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + statusCodeExpected: 401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + statusCodeExpected: 403 + }) + }) + + it('Should fail with a missing startDate query', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + statusCodeExpected: 400 + }) + }) + + it('Should fail with a bad startDate query', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: 'toto' }, + statusCodeExpected: 400 + }) + }) + + it('Should fail with a bad endDate query', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: new Date().toISOString(), endDate: 'toto' }, + statusCodeExpected: 400 + }) + }) + + it('Should fail with a bad level parameter', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: new Date().toISOString(), level: 'toto' }, + statusCodeExpected: 400 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: new Date().toISOString() }, + statusCodeExpected: 200 + }) + }) + }) + + after(async function () { + killallServers([ server ]) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts index 4e53074ab..94c15e0d0 100644 --- a/server/tests/api/server/index.ts +++ b/server/tests/api/server/index.ts @@ -6,6 +6,7 @@ import './follows' import './follows-moderation' import './handle-down' import './jobs' +import './logs' import './reverse-proxy' import './stats' import './tracker' diff --git a/server/tests/api/server/logs.ts b/server/tests/api/server/logs.ts new file mode 100644 index 000000000..05b0308de --- /dev/null +++ b/server/tests/api/server/logs.ts @@ -0,0 +1,92 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { flushTests, killallServers, runServer, ServerInfo, setAccessTokensToServers } from '../../../../shared/utils/index' +import { waitJobs } from '../../../../shared/utils/server/jobs' +import { uploadVideo } from '../../../../shared/utils/videos/videos' +import { getLogs } from '../../../../shared/utils/logs/logs' + +const expect = chai.expect + +describe('Test logs', function () { + let server: ServerInfo + + before(async function () { + this.timeout(30000) + + await flushTests() + + server = await runServer(1) + await setAccessTokensToServers([ server ]) + }) + + it('Should get logs with a start date', async function () { + this.timeout(10000) + + await uploadVideo(server.url, server.accessToken, { name: 'video 1' }) + await waitJobs([ server ]) + + const now = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 2' }) + await waitJobs([ server ]) + + const res = await getLogs(server.url, server.accessToken, now) + const logsString = JSON.stringify(res.body) + + expect(logsString.includes('video 1')).to.be.false + expect(logsString.includes('video 2')).to.be.true + }) + + it('Should get logs with an end date', async function () { + this.timeout(10000) + + await uploadVideo(server.url, server.accessToken, { name: 'video 3' }) + await waitJobs([ server ]) + + const now1 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 4' }) + await waitJobs([ server ]) + + const now2 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 5' }) + await waitJobs([ server ]) + + const res = await getLogs(server.url, server.accessToken, now1, now2) + const logsString = JSON.stringify(res.body) + + expect(logsString.includes('video 3')).to.be.false + expect(logsString.includes('video 4')).to.be.true + expect(logsString.includes('video 5')).to.be.false + }) + + it('Should get filter by level', async function () { + this.timeout(10000) + + const now = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 6' }) + await waitJobs([ server ]) + + { + const res = await getLogs(server.url, server.accessToken, now, undefined, 'info') + const logsString = JSON.stringify(res.body) + + expect(logsString.includes('video 6')).to.be.true + } + + { + const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn') + const logsString = JSON.stringify(res.body) + + expect(logsString.includes('video 6')).to.be.false + } + }) + + after(async function () { + killallServers([ server ]) + }) +}) diff --git a/shared/models/server/log-level.type.ts b/shared/models/server/log-level.type.ts new file mode 100644 index 000000000..ce91559e3 --- /dev/null +++ b/shared/models/server/log-level.type.ts @@ -0,0 +1 @@ +export type LogLevel = 'debug' | 'info' | 'warn' | 'error' diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts index eaa064bd9..5ec255ea5 100644 --- a/shared/models/users/user-right.enum.ts +++ b/shared/models/users/user-right.enum.ts @@ -5,6 +5,8 @@ export enum UserRight { MANAGE_SERVER_FOLLOW, + MANAGE_LOGS, + MANAGE_SERVER_REDUNDANCY, MANAGE_VIDEO_ABUSES, diff --git a/shared/utils/logs/logs.ts b/shared/utils/logs/logs.ts new file mode 100644 index 000000000..21adace82 --- /dev/null +++ b/shared/utils/logs/logs.ts @@ -0,0 +1,41 @@ +// Thanks: https://stackoverflow.com/a/37014317 +import { stat } from 'fs-extra' +import { makeGetRequest } from '../requests/requests' +import { LogLevel } from '../../models/server/log-level.type' + +async function mtimeSortFilesDesc (files: string[], basePath: string) { + const promises = [] + const out: { file: string, mtime: number }[] = [] + + for (const file of files) { + const p = stat(basePath + '/' + file) + .then(stats => { + if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() }) + }) + + promises.push(p) + } + + await Promise.all(promises) + + out.sort((a, b) => b.mtime - a.mtime) + + return out +} + +function getLogs (url: string, accessToken: string, startDate: Date, endDate?: Date, level?: LogLevel) { + const path = '/api/v1/server/logs' + + return makeGetRequest({ + url, + path, + token: accessToken, + query: { startDate, endDate, level }, + statusCodeExpected: 200 + }) +} + +export { + mtimeSortFilesDesc, + getLogs +} -- cgit v1.2.3