aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-04-10 15:26:33 +0200
committerChocobozzz <me@florianbigard.com>2019-04-10 16:38:32 +0200
commitfd8710b897a67518d3a61c319e54b6a65ba443ef (patch)
treed9953b7e0bb4e5a119c872ab21021f4c1ab33bea /server
parent31b6ddf86652502e0c96d77fa10861ce4af11aa4 (diff)
downloadPeerTube-fd8710b897a67518d3a61c319e54b6a65ba443ef.tar.gz
PeerTube-fd8710b897a67518d3a61c319e54b6a65ba443ef.tar.zst
PeerTube-fd8710b897a67518d3a61c319e54b6a65ba443ef.zip
Add logs endpoint
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/server/index.ts2
-rw-r--r--server/controllers/api/server/logs.ts90
-rw-r--r--server/helpers/custom-validators/logs.ts14
-rw-r--r--server/helpers/logger.ts11
-rw-r--r--server/initializers/constants.ts3
-rw-r--r--server/middlewares/validators/logs.ts31
-rw-r--r--server/middlewares/validators/videos/videos.ts8
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/logs.ts117
-rw-r--r--server/tests/api/server/index.ts1
-rw-r--r--server/tests/api/server/logs.ts92
11 files changed, 359 insertions, 11 deletions
diff --git a/server/controllers/api/server/index.ts b/server/controllers/api/server/index.ts
index 814248e5f..de09588df 100644
--- a/server/controllers/api/server/index.ts
+++ b/server/controllers/api/server/index.ts
@@ -4,6 +4,7 @@ import { statsRouter } from './stats'
4import { serverRedundancyRouter } from './redundancy' 4import { serverRedundancyRouter } from './redundancy'
5import { serverBlocklistRouter } from './server-blocklist' 5import { serverBlocklistRouter } from './server-blocklist'
6import { contactRouter } from './contact' 6import { contactRouter } from './contact'
7import { logsRouter } from './logs'
7 8
8const serverRouter = express.Router() 9const serverRouter = express.Router()
9 10
@@ -12,6 +13,7 @@ serverRouter.use('/', serverRedundancyRouter)
12serverRouter.use('/', statsRouter) 13serverRouter.use('/', statsRouter)
13serverRouter.use('/', serverBlocklistRouter) 14serverRouter.use('/', serverBlocklistRouter)
14serverRouter.use('/', contactRouter) 15serverRouter.use('/', contactRouter)
16serverRouter.use('/', logsRouter)
15 17
16// --------------------------------------------------------------------------- 18// ---------------------------------------------------------------------------
17 19
diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts
new file mode 100644
index 000000000..c551c67e3
--- /dev/null
+++ b/server/controllers/api/server/logs.ts
@@ -0,0 +1,90 @@
1import * as express from 'express'
2import { UserRight } from '../../../../shared/models/users'
3import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
4import { mtimeSortFilesDesc } from '../../../../shared/utils/logs/logs'
5import { readdir } from 'fs-extra'
6import { CONFIG, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers'
7import { createInterface } from 'readline'
8import { createReadStream } from 'fs'
9import { join } from 'path'
10import { getLogsValidator } from '../../../middlewares/validators/logs'
11import { LogLevel } from '../../../../shared/models/server/log-level.type'
12
13const logsRouter = express.Router()
14
15logsRouter.get('/logs',
16 authenticate,
17 ensureUserHasRight(UserRight.MANAGE_LOGS),
18 getLogsValidator,
19 asyncMiddleware(getLogs)
20)
21
22// ---------------------------------------------------------------------------
23
24export {
25 logsRouter
26}
27
28// ---------------------------------------------------------------------------
29
30async function getLogs (req: express.Request, res: express.Response) {
31 const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
32 const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR)
33 let currentSize = 0
34
35 const startDate = new Date(req.query.startDate)
36 const endDate = req.query.endDate ? new Date(req.query.endDate) : new Date()
37 const level: LogLevel = req.query.level || 'info'
38
39 let output = ''
40
41 for (const meta of sortedLogFiles) {
42 const path = join(CONFIG.STORAGE.LOG_DIR, meta.file)
43
44 const result = await getOutputFromFile(path, startDate, endDate, level, currentSize)
45 if (!result.output) break
46
47 output = output + result.output
48 currentSize = result.currentSize
49
50 if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) break
51 }
52
53 return res.json(output).end()
54}
55
56function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) {
57 const startTime = startDate.getTime()
58 const endTime = endDate.getTime()
59
60 const logsLevel: { [ id in LogLevel ]: number } = {
61 debug: 0,
62 info: 1,
63 warn: 2,
64 error: 3
65 }
66
67 return new Promise<{ output: string, currentSize: number }>(res => {
68 const stream = createReadStream(path)
69 let output = ''
70
71 stream.once('close', () => res({ output, currentSize }))
72
73 const rl = createInterface({
74 input: stream
75 })
76
77 rl.on('line', line => {
78 const log = JSON.parse(line)
79
80 const logTime = new Date(log.timestamp).getTime()
81 if (logTime >= startTime && logTime <= endTime && logsLevel[log.level] >= logsLevel[level]) {
82 output += line
83
84 currentSize += line.length
85
86 if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) stream.close()
87 }
88 })
89 })
90}
diff --git a/server/helpers/custom-validators/logs.ts b/server/helpers/custom-validators/logs.ts
new file mode 100644
index 000000000..30d0ce262
--- /dev/null
+++ b/server/helpers/custom-validators/logs.ts
@@ -0,0 +1,14 @@
1import { exists } from './misc'
2import { LogLevel } from '../../../shared/models/server/log-level.type'
3
4const logLevels: LogLevel[] = [ 'debug', 'info', 'warn', 'error' ]
5
6function isValidLogLevel (value: any) {
7 return exists(value) && logLevels.indexOf(value) !== -1
8}
9
10// ---------------------------------------------------------------------------
11
12export {
13 isValidLogLevel
14}
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 203e637a8..f8a142718 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -3,10 +3,12 @@ import { mkdirpSync } from 'fs-extra'
3import * as path from 'path' 3import * as path from 'path'
4import * as winston from 'winston' 4import * as winston from 'winston'
5import { CONFIG } from '../initializers' 5import { CONFIG } from '../initializers'
6import { omit } from 'lodash'
6 7
7const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT 8const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
8 9
9// Create the directory if it does not exist 10// Create the directory if it does not exist
11// FIXME: use async
10mkdirpSync(CONFIG.STORAGE.LOG_DIR) 12mkdirpSync(CONFIG.STORAGE.LOG_DIR)
11 13
12function loggerReplacer (key: string, value: any) { 14function loggerReplacer (key: string, value: any) {
@@ -22,13 +24,10 @@ function loggerReplacer (key: string, value: any) {
22} 24}
23 25
24const consoleLoggerFormat = winston.format.printf(info => { 26const consoleLoggerFormat = winston.format.printf(info => {
25 const obj = { 27 const obj = omit(info, 'label', 'timestamp', 'level', 'message')
26 meta: info.meta,
27 err: info.err,
28 sql: info.sql
29 }
30 28
31 let additionalInfos = JSON.stringify(obj, loggerReplacer, 2) 29 let additionalInfos = JSON.stringify(obj, loggerReplacer, 2)
30
32 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' 31 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
33 else additionalInfos = ' ' + additionalInfos 32 else additionalInfos = ' ' + additionalInfos
34 33
@@ -57,7 +56,7 @@ const logger = winston.createLogger({
57 filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube.log'), 56 filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube.log'),
58 handleExceptions: true, 57 handleExceptions: true,
59 maxsize: 1024 * 1024 * 12, 58 maxsize: 1024 * 1024 * 12,
60 maxFiles: 5, 59 maxFiles: 20,
61 format: winston.format.combine( 60 format: winston.format.combine(
62 winston.format.timestamp(), 61 winston.format.timestamp(),
63 jsonLoggerFormat 62 jsonLoggerFormat
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 3f02572db..739ea5502 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -730,6 +730,8 @@ const FEEDS = {
730 COUNT: 20 730 COUNT: 20
731} 731}
732 732
733const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000
734
733// --------------------------------------------------------------------------- 735// ---------------------------------------------------------------------------
734 736
735const TRACKER_RATE_LIMITS = { 737const TRACKER_RATE_LIMITS = {
@@ -819,6 +821,7 @@ export {
819 STATIC_PATHS, 821 STATIC_PATHS,
820 VIDEO_IMPORT_TIMEOUT, 822 VIDEO_IMPORT_TIMEOUT,
821 VIDEO_PLAYLIST_TYPES, 823 VIDEO_PLAYLIST_TYPES,
824 MAX_LOGS_OUTPUT_CHARACTERS,
822 ACTIVITY_PUB, 825 ACTIVITY_PUB,
823 ACTIVITY_PUB_ACTOR_TYPES, 826 ACTIVITY_PUB_ACTOR_TYPES,
824 THUMBNAILS_SIZE, 827 THUMBNAILS_SIZE,
diff --git a/server/middlewares/validators/logs.ts b/server/middlewares/validators/logs.ts
new file mode 100644
index 000000000..7380c6edd
--- /dev/null
+++ b/server/middlewares/validators/logs.ts
@@ -0,0 +1,31 @@
1import * as express from 'express'
2import { logger } from '../../helpers/logger'
3import { areValidationErrors } from './utils'
4import { isDateValid } from '../../helpers/custom-validators/misc'
5import { query } from 'express-validator/check'
6import { isValidLogLevel } from '../../helpers/custom-validators/logs'
7
8const getLogsValidator = [
9 query('startDate')
10 .custom(isDateValid).withMessage('Should have a valid start date'),
11 query('level')
12 .optional()
13 .custom(isValidLogLevel).withMessage('Should have a valid level'),
14 query('endDate')
15 .optional()
16 .custom(isDateValid).withMessage('Should have a valid end date'),
17
18 (req: express.Request, res: express.Response, next: express.NextFunction) => {
19 logger.debug('Checking getLogsValidator parameters.', { parameters: req.query })
20
21 if (areValidationErrors(req, res)) return
22
23 return next()
24 }
25]
26
27// ---------------------------------------------------------------------------
28
29export {
30 getLogsValidator
31}
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index b70abf429..e247db708 100644
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -14,18 +14,18 @@ import {
14} from '../../../helpers/custom-validators/misc' 14} from '../../../helpers/custom-validators/misc'
15import { 15import {
16 checkUserCanManageVideo, 16 checkUserCanManageVideo,
17 isVideoOriginallyPublishedAtValid, 17 doesVideoChannelOfAccountExist,
18 doesVideoExist,
18 isScheduleVideoUpdatePrivacyValid, 19 isScheduleVideoUpdatePrivacyValid,
19 isVideoCategoryValid, 20 isVideoCategoryValid,
20 doesVideoChannelOfAccountExist,
21 isVideoDescriptionValid, 21 isVideoDescriptionValid,
22 doesVideoExist,
23 isVideoFile, 22 isVideoFile,
24 isVideoFilterValid, 23 isVideoFilterValid,
25 isVideoImage, 24 isVideoImage,
26 isVideoLanguageValid, 25 isVideoLanguageValid,
27 isVideoLicenceValid, 26 isVideoLicenceValid,
28 isVideoNameValid, 27 isVideoNameValid,
28 isVideoOriginallyPublishedAtValid,
29 isVideoPrivacyValid, 29 isVideoPrivacyValid,
30 isVideoSupportValid, 30 isVideoSupportValid,
31 isVideoTagsValid 31 isVideoTagsValid
@@ -37,10 +37,8 @@ import { authenticatePromiseIfNeeded } from '../../oauth'
37import { areValidationErrors } from '../utils' 37import { areValidationErrors } from '../utils'
38import { cleanUpReqFiles } from '../../../helpers/express-utils' 38import { cleanUpReqFiles } from '../../../helpers/express-utils'
39import { VideoModel } from '../../../models/video/video' 39import { VideoModel } from '../../../models/video/video'
40import { UserModel } from '../../../models/account/user'
41import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } from '../../../helpers/custom-validators/video-ownership' 40import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } from '../../../helpers/custom-validators/video-ownership'
42import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model' 41import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model'
43import { VideoChangeOwnershipModel } from '../../../models/video/video-change-ownership'
44import { AccountModel } from '../../../models/account/account' 42import { AccountModel } from '../../../models/account/account'
45import { VideoFetchType } from '../../../helpers/video' 43import { VideoFetchType } from '../../../helpers/video'
46import { isNSFWQueryValid, isNumberArray, isStringArray } from '../../../helpers/custom-validators/search' 44import { isNSFWQueryValid, isNumberArray, isStringArray } from '../../../helpers/custom-validators/search'
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index ca51cd39a..bdac95025 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -4,6 +4,7 @@ import './config'
4import './contact-form' 4import './contact-form'
5import './follows' 5import './follows'
6import './jobs' 6import './jobs'
7import './logs'
7import './redundancy' 8import './redundancy'
8import './search' 9import './search'
9import './services' 10import './services'
diff --git a/server/tests/api/check-params/logs.ts b/server/tests/api/check-params/logs.ts
new file mode 100644
index 000000000..d6a40da61
--- /dev/null
+++ b/server/tests/api/check-params/logs.ts
@@ -0,0 +1,117 @@
1/* tslint:disable:no-unused-expression */
2
3import 'mocha'
4
5import {
6 createUser,
7 flushTests,
8 killallServers,
9 runServer,
10 ServerInfo,
11 setAccessTokensToServers,
12 userLogin
13} from '../../../../shared/utils'
14import { makeGetRequest } from '../../../../shared/utils/requests/requests'
15
16describe('Test logs API validators', function () {
17 const path = '/api/v1/server/logs'
18 let server: ServerInfo
19 let userAccessToken = ''
20
21 // ---------------------------------------------------------------
22
23 before(async function () {
24 this.timeout(120000)
25
26 await flushTests()
27
28 server = await runServer(1)
29
30 await setAccessTokensToServers([ server ])
31
32 const user = {
33 username: 'user1',
34 password: 'my super password'
35 }
36 await createUser(server.url, server.accessToken, user.username, user.password)
37 userAccessToken = await userLogin(server, user)
38 })
39
40 describe('When getting logs', function () {
41
42 it('Should fail with a non authenticated user', async function () {
43 await makeGetRequest({
44 url: server.url,
45 path,
46 statusCodeExpected: 401
47 })
48 })
49
50 it('Should fail with a non admin user', async function () {
51 await makeGetRequest({
52 url: server.url,
53 path,
54 token: userAccessToken,
55 statusCodeExpected: 403
56 })
57 })
58
59 it('Should fail with a missing startDate query', async function () {
60 await makeGetRequest({
61 url: server.url,
62 path,
63 token: server.accessToken,
64 statusCodeExpected: 400
65 })
66 })
67
68 it('Should fail with a bad startDate query', async function () {
69 await makeGetRequest({
70 url: server.url,
71 path,
72 token: server.accessToken,
73 query: { startDate: 'toto' },
74 statusCodeExpected: 400
75 })
76 })
77
78 it('Should fail with a bad endDate query', async function () {
79 await makeGetRequest({
80 url: server.url,
81 path,
82 token: server.accessToken,
83 query: { startDate: new Date().toISOString(), endDate: 'toto' },
84 statusCodeExpected: 400
85 })
86 })
87
88 it('Should fail with a bad level parameter', async function () {
89 await makeGetRequest({
90 url: server.url,
91 path,
92 token: server.accessToken,
93 query: { startDate: new Date().toISOString(), level: 'toto' },
94 statusCodeExpected: 400
95 })
96 })
97
98 it('Should succeed with the correct params', async function () {
99 await makeGetRequest({
100 url: server.url,
101 path,
102 token: server.accessToken,
103 query: { startDate: new Date().toISOString() },
104 statusCodeExpected: 200
105 })
106 })
107 })
108
109 after(async function () {
110 killallServers([ server ])
111
112 // Keep the logs if the test failed
113 if (this['ok']) {
114 await flushTests()
115 }
116 })
117})
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts
index 4e53074ab..94c15e0d0 100644
--- a/server/tests/api/server/index.ts
+++ b/server/tests/api/server/index.ts
@@ -6,6 +6,7 @@ import './follows'
6import './follows-moderation' 6import './follows-moderation'
7import './handle-down' 7import './handle-down'
8import './jobs' 8import './jobs'
9import './logs'
9import './reverse-proxy' 10import './reverse-proxy'
10import './stats' 11import './stats'
11import './tracker' 12import './tracker'
diff --git a/server/tests/api/server/logs.ts b/server/tests/api/server/logs.ts
new file mode 100644
index 000000000..05b0308de
--- /dev/null
+++ b/server/tests/api/server/logs.ts
@@ -0,0 +1,92 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import { flushTests, killallServers, runServer, ServerInfo, setAccessTokensToServers } from '../../../../shared/utils/index'
6import { waitJobs } from '../../../../shared/utils/server/jobs'
7import { uploadVideo } from '../../../../shared/utils/videos/videos'
8import { getLogs } from '../../../../shared/utils/logs/logs'
9
10const expect = chai.expect
11
12describe('Test logs', function () {
13 let server: ServerInfo
14
15 before(async function () {
16 this.timeout(30000)
17
18 await flushTests()
19
20 server = await runServer(1)
21 await setAccessTokensToServers([ server ])
22 })
23
24 it('Should get logs with a start date', async function () {
25 this.timeout(10000)
26
27 await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
28 await waitJobs([ server ])
29
30 const now = new Date()
31
32 await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
33 await waitJobs([ server ])
34
35 const res = await getLogs(server.url, server.accessToken, now)
36 const logsString = JSON.stringify(res.body)
37
38 expect(logsString.includes('video 1')).to.be.false
39 expect(logsString.includes('video 2')).to.be.true
40 })
41
42 it('Should get logs with an end date', async function () {
43 this.timeout(10000)
44
45 await uploadVideo(server.url, server.accessToken, { name: 'video 3' })
46 await waitJobs([ server ])
47
48 const now1 = new Date()
49
50 await uploadVideo(server.url, server.accessToken, { name: 'video 4' })
51 await waitJobs([ server ])
52
53 const now2 = new Date()
54
55 await uploadVideo(server.url, server.accessToken, { name: 'video 5' })
56 await waitJobs([ server ])
57
58 const res = await getLogs(server.url, server.accessToken, now1, now2)
59 const logsString = JSON.stringify(res.body)
60
61 expect(logsString.includes('video 3')).to.be.false
62 expect(logsString.includes('video 4')).to.be.true
63 expect(logsString.includes('video 5')).to.be.false
64 })
65
66 it('Should get filter by level', async function () {
67 this.timeout(10000)
68
69 const now = new Date()
70
71 await uploadVideo(server.url, server.accessToken, { name: 'video 6' })
72 await waitJobs([ server ])
73
74 {
75 const res = await getLogs(server.url, server.accessToken, now, undefined, 'info')
76 const logsString = JSON.stringify(res.body)
77
78 expect(logsString.includes('video 6')).to.be.true
79 }
80
81 {
82 const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn')
83 const logsString = JSON.stringify(res.body)
84
85 expect(logsString.includes('video 6')).to.be.false
86 }
87 })
88
89 after(async function () {
90 killallServers([ server ])
91 })
92})