+++ /dev/null
-import { Component } from '@angular/core'
-
-@Component({
- template: '<router-outlet></router-outlet>'
-})
-export class JobsComponent {}
+++ /dev/null
-import { Routes } from '@angular/router'
-import { UserRight } from '../../../../../shared'
-import { UserRightGuard } from '../../core'
-import { JobsComponent } from './job.component'
-import { JobsListComponent } from './jobs-list/jobs-list.component'
-
-export const JobsRoutes: Routes = [
- {
- path: 'jobs',
- component: JobsComponent,
- canActivate: [ UserRightGuard ],
- data: {
- userRight: UserRight.MANAGE_JOBS
- },
- children: [
- {
- path: '',
- redirectTo: 'list',
- pathMatch: 'full'
- },
- {
- path: 'list',
- component: JobsListComponent,
- data: {
- meta: {
- title: 'Jobs list'
- }
- }
- }
- ]
- }
-]
+++ /dev/null
-export * from './jobs-list.component'
+++ /dev/null
-export * from './job.service'
import * as program from 'commander'
-import { createReadStream, readdirSync, statSync } from 'fs-extra'
+import { createReadStream, readdir } from 'fs-extra'
import { join } from 'path'
import { createInterface } from 'readline'
import * as winston from 'winston'
import { labelFormatter } from '../server/helpers/logger'
import { CONFIG } from '../server/initializers/constants'
+import { mtimeSortFilesDesc } from '../shared/utils/logs/logs'
program
.option('-l, --level [level]', 'Level log (debug/info/warn/error)')
debug: logger.debug.bind(logger)
}
-const logFiles = readdirSync(CONFIG.STORAGE.LOG_DIR)
-const lastLogFile = getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR)
+run()
+ .then(() => process.exit(0))
+ .catch(err => console.error(err))
-const path = join(CONFIG.STORAGE.LOG_DIR, lastLogFile)
-console.log('Opening %s.', path)
+function run () {
+ return new Promise(async res => {
+ const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
+ const lastLogFile = await getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR)
-const rl = createInterface({
- input: createReadStream(path)
-})
+ const path = join(CONFIG.STORAGE.LOG_DIR, lastLogFile)
+ console.log('Opening %s.', path)
-rl.on('line', line => {
- const log = JSON.parse(line)
- // Don't know why but loggerFormat does not remove splat key
- Object.assign(log, { splat: undefined })
+ const stream = createReadStream(path)
- logLevels[log.level](log)
-})
+ const rl = createInterface({
+ input: stream
+ })
-function toTimeFormat (time: string) {
- const timestamp = Date.parse(time)
+ rl.on('line', line => {
+ const log = JSON.parse(line)
+ // Don't know why but loggerFormat does not remove splat key
+ Object.assign(log, { splat: undefined })
- if (isNaN(timestamp) === true) return 'Unknown date'
+ logLevels[ log.level ](log)
+ })
- return new Date(timestamp).toISOString()
+ stream.once('close', () => res())
+ })
}
// Thanks: https://stackoverflow.com/a/37014317
-function getNewestFile (files: string[], basePath: string) {
- const out = []
+async function getNewestFile (files: string[], basePath: string) {
+ const sorted = await mtimeSortFilesDesc(files, basePath)
- files.forEach(file => {
- const stats = statSync(basePath + '/' + file)
- if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() })
- })
+ return (sorted.length > 0) ? sorted[ 0 ].file : ''
+}
+
+function toTimeFormat (time: string) {
+ const timestamp = Date.parse(time)
- out.sort((a, b) => b.mtime - a.mtime)
+ if (isNaN(timestamp) === true) return 'Unknown date'
- return (out.length > 0) ? out[ 0 ].file : ''
+ return new Date(timestamp).toISOString()
}
import { serverRedundancyRouter } from './redundancy'
import { serverBlocklistRouter } from './server-blocklist'
import { contactRouter } from './contact'
+import { logsRouter } from './logs'
const serverRouter = express.Router()
serverRouter.use('/', statsRouter)
serverRouter.use('/', serverBlocklistRouter)
serverRouter.use('/', contactRouter)
+serverRouter.use('/', logsRouter)
// ---------------------------------------------------------------------------
--- /dev/null
+import * as express from 'express'
+import { UserRight } from '../../../../shared/models/users'
+import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
+import { mtimeSortFilesDesc } from '../../../../shared/utils/logs/logs'
+import { readdir } from 'fs-extra'
+import { CONFIG, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers'
+import { createInterface } from 'readline'
+import { createReadStream } from 'fs'
+import { join } from 'path'
+import { getLogsValidator } from '../../../middlewares/validators/logs'
+import { LogLevel } from '../../../../shared/models/server/log-level.type'
+
+const logsRouter = express.Router()
+
+logsRouter.get('/logs',
+ authenticate,
+ ensureUserHasRight(UserRight.MANAGE_LOGS),
+ getLogsValidator,
+ asyncMiddleware(getLogs)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+ logsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function getLogs (req: express.Request, res: express.Response) {
+ const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
+ const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR)
+ let currentSize = 0
+
+ const startDate = new Date(req.query.startDate)
+ const endDate = req.query.endDate ? new Date(req.query.endDate) : new Date()
+ const level: LogLevel = req.query.level || 'info'
+
+ let output = ''
+
+ for (const meta of sortedLogFiles) {
+ const path = join(CONFIG.STORAGE.LOG_DIR, meta.file)
+
+ const result = await getOutputFromFile(path, startDate, endDate, level, currentSize)
+ if (!result.output) break
+
+ output = output + result.output
+ currentSize = result.currentSize
+
+ if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) break
+ }
+
+ return res.json(output).end()
+}
+
+function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) {
+ const startTime = startDate.getTime()
+ const endTime = endDate.getTime()
+
+ const logsLevel: { [ id in LogLevel ]: number } = {
+ debug: 0,
+ info: 1,
+ warn: 2,
+ error: 3
+ }
+
+ return new Promise<{ output: string, currentSize: number }>(res => {
+ const stream = createReadStream(path)
+ let output = ''
+
+ stream.once('close', () => res({ output, currentSize }))
+
+ const rl = createInterface({
+ input: stream
+ })
+
+ rl.on('line', line => {
+ const log = JSON.parse(line)
+
+ const logTime = new Date(log.timestamp).getTime()
+ if (logTime >= startTime && logTime <= endTime && logsLevel[log.level] >= logsLevel[level]) {
+ output += line
+
+ currentSize += line.length
+
+ if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) stream.close()
+ }
+ })
+ })
+}
--- /dev/null
+import { exists } from './misc'
+import { LogLevel } from '../../../shared/models/server/log-level.type'
+
+const logLevels: LogLevel[] = [ 'debug', 'info', 'warn', 'error' ]
+
+function isValidLogLevel (value: any) {
+ return exists(value) && logLevels.indexOf(value) !== -1
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+ isValidLogLevel
+}
import * as path from 'path'
import * as winston from 'winston'
import { CONFIG } from '../initializers'
+import { omit } from 'lodash'
const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
// Create the directory if it does not exist
+// FIXME: use async
mkdirpSync(CONFIG.STORAGE.LOG_DIR)
function loggerReplacer (key: string, value: any) {
}
const consoleLoggerFormat = winston.format.printf(info => {
- const obj = {
- meta: info.meta,
- err: info.err,
- sql: info.sql
- }
+ const obj = omit(info, 'label', 'timestamp', 'level', 'message')
let additionalInfos = JSON.stringify(obj, loggerReplacer, 2)
+
if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
else additionalInfos = ' ' + additionalInfos
filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube.log'),
handleExceptions: true,
maxsize: 1024 * 1024 * 12,
- maxFiles: 5,
+ maxFiles: 20,
format: winston.format.combine(
winston.format.timestamp(),
jsonLoggerFormat
COUNT: 20
}
+const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000
+
// ---------------------------------------------------------------------------
const TRACKER_RATE_LIMITS = {
STATIC_PATHS,
VIDEO_IMPORT_TIMEOUT,
VIDEO_PLAYLIST_TYPES,
+ MAX_LOGS_OUTPUT_CHARACTERS,
ACTIVITY_PUB,
ACTIVITY_PUB_ACTOR_TYPES,
THUMBNAILS_SIZE,
--- /dev/null
+import * as express from 'express'
+import { logger } from '../../helpers/logger'
+import { areValidationErrors } from './utils'
+import { isDateValid } from '../../helpers/custom-validators/misc'
+import { query } from 'express-validator/check'
+import { isValidLogLevel } from '../../helpers/custom-validators/logs'
+
+const getLogsValidator = [
+ query('startDate')
+ .custom(isDateValid).withMessage('Should have a valid start date'),
+ query('level')
+ .optional()
+ .custom(isValidLogLevel).withMessage('Should have a valid level'),
+ query('endDate')
+ .optional()
+ .custom(isDateValid).withMessage('Should have a valid end date'),
+
+ (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ logger.debug('Checking getLogsValidator parameters.', { parameters: req.query })
+
+ if (areValidationErrors(req, res)) return
+
+ return next()
+ }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+ getLogsValidator
+}
} from '../../../helpers/custom-validators/misc'
import {
checkUserCanManageVideo,
- isVideoOriginallyPublishedAtValid,
+ doesVideoChannelOfAccountExist,
+ doesVideoExist,
isScheduleVideoUpdatePrivacyValid,
isVideoCategoryValid,
- doesVideoChannelOfAccountExist,
isVideoDescriptionValid,
- doesVideoExist,
isVideoFile,
isVideoFilterValid,
isVideoImage,
isVideoLanguageValid,
isVideoLicenceValid,
isVideoNameValid,
+ isVideoOriginallyPublishedAtValid,
isVideoPrivacyValid,
isVideoSupportValid,
isVideoTagsValid
import { areValidationErrors } from '../utils'
import { cleanUpReqFiles } from '../../../helpers/express-utils'
import { VideoModel } from '../../../models/video/video'
-import { UserModel } from '../../../models/account/user'
import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } from '../../../helpers/custom-validators/video-ownership'
import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model'
-import { VideoChangeOwnershipModel } from '../../../models/video/video-change-ownership'
import { AccountModel } from '../../../models/account/account'
import { VideoFetchType } from '../../../helpers/video'
import { isNSFWQueryValid, isNumberArray, isStringArray } from '../../../helpers/custom-validators/search'
import './contact-form'
import './follows'
import './jobs'
+import './logs'
import './redundancy'
import './search'
import './services'
--- /dev/null
+/* tslint:disable:no-unused-expression */
+
+import 'mocha'
+
+import {
+ createUser,
+ flushTests,
+ killallServers,
+ runServer,
+ ServerInfo,
+ setAccessTokensToServers,
+ userLogin
+} from '../../../../shared/utils'
+import { makeGetRequest } from '../../../../shared/utils/requests/requests'
+
+describe('Test logs API validators', function () {
+ const path = '/api/v1/server/logs'
+ let server: ServerInfo
+ let userAccessToken = ''
+
+ // ---------------------------------------------------------------
+
+ before(async function () {
+ this.timeout(120000)
+
+ await flushTests()
+
+ server = await runServer(1)
+
+ await setAccessTokensToServers([ server ])
+
+ const user = {
+ username: 'user1',
+ password: 'my super password'
+ }
+ await createUser(server.url, server.accessToken, user.username, user.password)
+ userAccessToken = await userLogin(server, user)
+ })
+
+ describe('When getting logs', function () {
+
+ it('Should fail with a non authenticated user', async function () {
+ await makeGetRequest({
+ url: server.url,
+ path,
+ statusCodeExpected: 401
+ })
+ })
+
+ it('Should fail with a non admin user', async function () {
+ await makeGetRequest({
+ url: server.url,
+ path,
+ token: userAccessToken,
+ statusCodeExpected: 403
+ })
+ })
+
+ it('Should fail with a missing startDate query', async function () {
+ await makeGetRequest({
+ url: server.url,
+ path,
+ token: server.accessToken,
+ statusCodeExpected: 400
+ })
+ })
+
+ it('Should fail with a bad startDate query', async function () {
+ await makeGetRequest({
+ url: server.url,
+ path,
+ token: server.accessToken,
+ query: { startDate: 'toto' },
+ statusCodeExpected: 400
+ })
+ })
+
+ it('Should fail with a bad endDate query', async function () {
+ await makeGetRequest({
+ url: server.url,
+ path,
+ token: server.accessToken,
+ query: { startDate: new Date().toISOString(), endDate: 'toto' },
+ statusCodeExpected: 400
+ })
+ })
+
+ it('Should fail with a bad level parameter', async function () {
+ await makeGetRequest({
+ url: server.url,
+ path,
+ token: server.accessToken,
+ query: { startDate: new Date().toISOString(), level: 'toto' },
+ statusCodeExpected: 400
+ })
+ })
+
+ it('Should succeed with the correct params', async function () {
+ await makeGetRequest({
+ url: server.url,
+ path,
+ token: server.accessToken,
+ query: { startDate: new Date().toISOString() },
+ statusCodeExpected: 200
+ })
+ })
+ })
+
+ after(async function () {
+ killallServers([ server ])
+
+ // Keep the logs if the test failed
+ if (this['ok']) {
+ await flushTests()
+ }
+ })
+})
import './follows-moderation'
import './handle-down'
import './jobs'
+import './logs'
import './reverse-proxy'
import './stats'
import './tracker'
--- /dev/null
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import { flushTests, killallServers, runServer, ServerInfo, setAccessTokensToServers } from '../../../../shared/utils/index'
+import { waitJobs } from '../../../../shared/utils/server/jobs'
+import { uploadVideo } from '../../../../shared/utils/videos/videos'
+import { getLogs } from '../../../../shared/utils/logs/logs'
+
+const expect = chai.expect
+
+describe('Test logs', function () {
+ let server: ServerInfo
+
+ before(async function () {
+ this.timeout(30000)
+
+ await flushTests()
+
+ server = await runServer(1)
+ await setAccessTokensToServers([ server ])
+ })
+
+ it('Should get logs with a start date', async function () {
+ this.timeout(10000)
+
+ await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
+ await waitJobs([ server ])
+
+ const now = new Date()
+
+ await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
+ await waitJobs([ server ])
+
+ const res = await getLogs(server.url, server.accessToken, now)
+ const logsString = JSON.stringify(res.body)
+
+ expect(logsString.includes('video 1')).to.be.false
+ expect(logsString.includes('video 2')).to.be.true
+ })
+
+ it('Should get logs with an end date', async function () {
+ this.timeout(10000)
+
+ await uploadVideo(server.url, server.accessToken, { name: 'video 3' })
+ await waitJobs([ server ])
+
+ const now1 = new Date()
+
+ await uploadVideo(server.url, server.accessToken, { name: 'video 4' })
+ await waitJobs([ server ])
+
+ const now2 = new Date()
+
+ await uploadVideo(server.url, server.accessToken, { name: 'video 5' })
+ await waitJobs([ server ])
+
+ const res = await getLogs(server.url, server.accessToken, now1, now2)
+ const logsString = JSON.stringify(res.body)
+
+ expect(logsString.includes('video 3')).to.be.false
+ expect(logsString.includes('video 4')).to.be.true
+ expect(logsString.includes('video 5')).to.be.false
+ })
+
+ it('Should get filter by level', async function () {
+ this.timeout(10000)
+
+ const now = new Date()
+
+ await uploadVideo(server.url, server.accessToken, { name: 'video 6' })
+ await waitJobs([ server ])
+
+ {
+ const res = await getLogs(server.url, server.accessToken, now, undefined, 'info')
+ const logsString = JSON.stringify(res.body)
+
+ expect(logsString.includes('video 6')).to.be.true
+ }
+
+ {
+ const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn')
+ const logsString = JSON.stringify(res.body)
+
+ expect(logsString.includes('video 6')).to.be.false
+ }
+ })
+
+ after(async function () {
+ killallServers([ server ])
+ })
+})
--- /dev/null
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error'
MANAGE_SERVER_FOLLOW,
+ MANAGE_LOGS,
+
MANAGE_SERVER_REDUNDANCY,
MANAGE_VIDEO_ABUSES,
--- /dev/null
+// Thanks: https://stackoverflow.com/a/37014317
+import { stat } from 'fs-extra'
+import { makeGetRequest } from '../requests/requests'
+import { LogLevel } from '../../models/server/log-level.type'
+
+async function mtimeSortFilesDesc (files: string[], basePath: string) {
+ const promises = []
+ const out: { file: string, mtime: number }[] = []
+
+ for (const file of files) {
+ const p = stat(basePath + '/' + file)
+ .then(stats => {
+ if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() })
+ })
+
+ promises.push(p)
+ }
+
+ await Promise.all(promises)
+
+ out.sort((a, b) => b.mtime - a.mtime)
+
+ return out
+}
+
+function getLogs (url: string, accessToken: string, startDate: Date, endDate?: Date, level?: LogLevel) {
+ const path = '/api/v1/server/logs'
+
+ return makeGetRequest({
+ url,
+ path,
+ token: accessToken,
+ query: { startDate, endDate, level },
+ statusCodeExpected: 200
+ })
+}
+
+export {
+ mtimeSortFilesDesc,
+ getLogs
+}
projects / github / Chocobozzz / PeerTube.git / commitdiff
