aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-03-05 13:26:02 +0100
committerChocobozzz <me@florianbigard.com>2021-03-24 18:18:40 +0100
commit452b3bea082481b84537e55c7cedc1e24860d543 (patch)
treeedd6a3eb988b093abbcde9fd1890b7fa69d8f25f
parentd00717523890dfc90859a27921f20d79d101ae6e (diff)
downloadPeerTube-452b3bea082481b84537e55c7cedc1e24860d543.tar.gz
PeerTube-452b3bea082481b84537e55c7cedc1e24860d543.tar.zst
PeerTube-452b3bea082481b84537e55c7cedc1e24860d543.zip
Introduce tags to the logger
That could help to filter unneeded logs and/or select particular components
-rwxr-xr-xscripts/parse-log.ts23
-rw-r--r--server.ts4
-rw-r--r--server/controllers/api/videos/index.ts11
-rw-r--r--server/helpers/logger.ts10
-rw-r--r--server/initializers/database.ts2
-rw-r--r--server/lib/activitypub/send/send-create.ts10
-rw-r--r--server/lib/activitypub/share.ts23
-rw-r--r--server/lib/video-blacklist.ts6
8 files changed, 66 insertions, 23 deletions
diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts
index 3679dab74..5f4480c88 100755
--- a/scripts/parse-log.ts
+++ b/scripts/parse-log.ts
@@ -15,6 +15,8 @@ import { format as sqlFormat } from 'sql-formatter'
15program 15program
16 .option('-l, --level [level]', 'Level log (debug/info/warn/error)') 16 .option('-l, --level [level]', 'Level log (debug/info/warn/error)')
17 .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)') 17 .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)')
18 .option('-t, --tags [tags...]', 'Display only lines with these tags')
19 .option('-nt, --not-tags [tags...]', 'Donrt display lines containing these tags')
18 .parse(process.argv) 20 .parse(process.argv)
19 21
20const options = program.opts() 22const options = program.opts()
@@ -24,6 +26,7 @@ const excludedKeys = {
24 message: true, 26 message: true,
25 splat: true, 27 splat: true,
26 timestamp: true, 28 timestamp: true,
29 tags: true,
27 label: true, 30 label: true,
28 sql: true 31 sql: true
29} 32}
@@ -93,6 +96,14 @@ function run () {
93 rl.on('line', line => { 96 rl.on('line', line => {
94 try { 97 try {
95 const log = JSON.parse(line) 98 const log = JSON.parse(line)
99 if (options.tags && !containsTags(log.tags, options.tags)) {
100 return
101 }
102
103 if (options.notTags && containsTags(log.tags, options.notTags)) {
104 return
105 }
106
96 // Don't know why but loggerFormat does not remove splat key 107 // Don't know why but loggerFormat does not remove splat key
97 Object.assign(log, { splat: undefined }) 108 Object.assign(log, { splat: undefined })
98 109
@@ -131,3 +142,15 @@ function toTimeFormat (time: string) {
131 142
132 return new Date(timestamp).toISOString() 143 return new Date(timestamp).toISOString()
133} 144}
145
146function containsTags (loggerTags: string[], optionsTags: string[]) {
147 if (!loggerTags) return false
148
149 for (const lt of loggerTags) {
150 for (const ot of optionsTags) {
151 if (lt === ot) return true
152 }
153 }
154
155 return false
156}
diff --git a/server.ts b/server.ts
index 00cd87e20..1cc29209e 100644
--- a/server.ts
+++ b/server.ts
@@ -160,7 +160,9 @@ morgan.token('user-agent', (req: express.Request) => {
160 return req.get('user-agent') 160 return req.get('user-agent')
161}) 161})
162app.use(morgan('combined', { 162app.use(morgan('combined', {
163 stream: { write: logger.info.bind(logger) }, 163 stream: {
164 write: (str: string) => logger.info(str, { tags: [ 'http' ] })
165 },
164 skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping' 166 skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping'
165})) 167}))
166 168
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 2447c1288..7fee278f2 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -17,7 +17,7 @@ import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../
17import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' 17import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
18import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils' 18import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils'
19import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' 19import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
20import { logger } from '../../../helpers/logger' 20import { logger, loggerTagsFactory } from '../../../helpers/logger'
21import { getFormattedObjects } from '../../../helpers/utils' 21import { getFormattedObjects } from '../../../helpers/utils'
22import { CONFIG } from '../../../initializers/config' 22import { CONFIG } from '../../../initializers/config'
23import { 23import {
@@ -67,6 +67,7 @@ import { ownershipVideoRouter } from './ownership'
67import { rateVideoRouter } from './rate' 67import { rateVideoRouter } from './rate'
68import { watchingRouter } from './watching' 68import { watchingRouter } from './watching'
69 69
70const lTags = loggerTagsFactory('api', 'video')
70const auditLogger = auditLoggerFactory('videos') 71const auditLogger = auditLoggerFactory('videos')
71const videosRouter = express.Router() 72const videosRouter = express.Router()
72 73
@@ -257,14 +258,14 @@ async function addVideo (req: express.Request, res: express.Response) {
257 }) 258 })
258 259
259 auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) 260 auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
260 logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) 261 logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid, lTags(videoCreated.uuid))
261 262
262 return { videoCreated } 263 return { videoCreated }
263 }) 264 })
264 265
265 // Create the torrent file in async way because it could be long 266 // Create the torrent file in async way because it could be long
266 createTorrentAndSetInfoHashAsync(video, videoFile) 267 createTorrentAndSetInfoHashAsync(video, videoFile)
267 .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err })) 268 .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) }))
268 .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)) 269 .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id))
269 .then(refreshedVideo => { 270 .then(refreshedVideo => {
270 if (!refreshedVideo) return 271 if (!refreshedVideo) return
@@ -276,7 +277,7 @@ async function addVideo (req: express.Request, res: express.Response) {
276 return sequelizeTypescript.transaction(t => federateVideoIfNeeded(refreshedVideo, true, t)) 277 return sequelizeTypescript.transaction(t => federateVideoIfNeeded(refreshedVideo, true, t))
277 }) 278 })
278 }) 279 })
279 .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err })) 280 .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) }))
280 281
281 if (video.state === VideoState.TO_TRANSCODE) { 282 if (video.state === VideoState.TO_TRANSCODE) {
282 await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User) 283 await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User)
@@ -389,7 +390,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
389 new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()), 390 new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()),
390 oldVideoAuditView 391 oldVideoAuditView
391 ) 392 )
392 logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid) 393 logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid, lTags(videoInstance.uuid))
393 394
394 return videoInstanceUpdated 395 return videoInstanceUpdated
395 }) 396 })
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 6917a64d9..a112fd300 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -48,7 +48,7 @@ function getLoggerReplacer () {
48} 48}
49 49
50const consoleLoggerFormat = winston.format.printf(info => { 50const consoleLoggerFormat = winston.format.printf(info => {
51 const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql' ] 51 const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ]
52 52
53 const obj = omit(info, ...toOmit) 53 const obj = omit(info, ...toOmit)
54 54
@@ -150,6 +150,13 @@ const bunyanLogger = {
150 error: bunyanLogFactory('error'), 150 error: bunyanLogFactory('error'),
151 fatal: bunyanLogFactory('error') 151 fatal: bunyanLogFactory('error')
152} 152}
153
154function loggerTagsFactory (...defaultTags: string[]) {
155 return (...tags: string[]) => {
156 return { tags: defaultTags.concat(tags) }
157 }
158}
159
153// --------------------------------------------------------------------------- 160// ---------------------------------------------------------------------------
154 161
155export { 162export {
@@ -159,5 +166,6 @@ export {
159 consoleLoggerFormat, 166 consoleLoggerFormat,
160 jsonLoggerFormat, 167 jsonLoggerFormat,
161 logger, 168 logger,
169 loggerTagsFactory,
162 bunyanLogger 170 bunyanLogger
163} 171}
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 1f2b6d521..8378fa982 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -76,7 +76,7 @@ const sequelizeTypescript = new SequelizeTypescript({
76 newMessage += ' in ' + benchmark + 'ms' 76 newMessage += ' in ' + benchmark + 'ms'
77 } 77 }
78 78
79 logger.debug(newMessage, { sql: message }) 79 logger.debug(newMessage, { sql: message, tags: [ 'sql' ] })
80 } 80 }
81}) 81})
82 82
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 9fb218224..baded642a 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -4,7 +4,7 @@ import { VideoPrivacy } from '../../../../shared/models/videos'
4import { VideoCommentModel } from '../../../models/video/video-comment' 4import { VideoCommentModel } from '../../../models/video/video-comment'
5import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' 5import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
6import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' 6import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience'
7import { logger } from '../../../helpers/logger' 7import { logger, loggerTagsFactory } from '../../../helpers/logger'
8import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' 8import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
9import { 9import {
10 MActorLight, 10 MActorLight,
@@ -18,10 +18,12 @@ import {
18import { getServerActor } from '@server/models/application/application' 18import { getServerActor } from '@server/models/application/application'
19import { ContextType } from '@shared/models/activitypub/context' 19import { ContextType } from '@shared/models/activitypub/context'
20 20
21const lTags = loggerTagsFactory('ap', 'create')
22
21async function sendCreateVideo (video: MVideoAP, t: Transaction) { 23async function sendCreateVideo (video: MVideoAP, t: Transaction) {
22 if (!video.hasPrivacyForFederation()) return undefined 24 if (!video.hasPrivacyForFederation()) return undefined
23 25
24 logger.info('Creating job to send video creation of %s.', video.url) 26 logger.info('Creating job to send video creation of %s.', video.url, lTags(video.uuid))
25 27
26 const byActor = video.VideoChannel.Account.Actor 28 const byActor = video.VideoChannel.Account.Actor
27 const videoObject = video.toActivityPubObject() 29 const videoObject = video.toActivityPubObject()
@@ -37,7 +39,7 @@ async function sendCreateCacheFile (
37 video: MVideoAccountLight, 39 video: MVideoAccountLight,
38 fileRedundancy: MVideoRedundancyStreamingPlaylistVideo | MVideoRedundancyFileVideo 40 fileRedundancy: MVideoRedundancyStreamingPlaylistVideo | MVideoRedundancyFileVideo
39) { 41) {
40 logger.info('Creating job to send file cache of %s.', fileRedundancy.url) 42 logger.info('Creating job to send file cache of %s.', fileRedundancy.url, lTags(video.uuid))
41 43
42 return sendVideoRelatedCreateActivity({ 44 return sendVideoRelatedCreateActivity({
43 byActor, 45 byActor,
@@ -51,7 +53,7 @@ async function sendCreateCacheFile (
51async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, t: Transaction) { 53async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, t: Transaction) {
52 if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined 54 if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined
53 55
54 logger.info('Creating job to send create video playlist of %s.', playlist.url) 56 logger.info('Creating job to send create video playlist of %s.', playlist.url, lTags(playlist.uuid))
55 57
56 const byActor = playlist.OwnerAccount.Actor 58 const byActor = playlist.OwnerAccount.Actor
57 const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC) 59 const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC)
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index 1f8a8f3c4..dde0c628e 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -1,15 +1,17 @@
1import { Transaction } from 'sequelize'
2import { VideoShareModel } from '../../models/video/video-share'
3import { sendUndoAnnounce, sendVideoAnnounce } from './send'
4import { getLocalVideoAnnounceActivityPubUrl } from './url'
5import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import { Transaction } from 'sequelize'
3import { getServerActor } from '@server/models/application/application'
4import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
5import { logger, loggerTagsFactory } from '../../helpers/logger'
6import { doRequest } from '../../helpers/requests' 6import { doRequest } from '../../helpers/requests'
7import { getOrCreateActorAndServerAndModel } from './actor'
8import { logger } from '../../helpers/logger'
9import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
10import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 8import { VideoShareModel } from '../../models/video/video-share'
11import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' 9import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video'
12import { getServerActor } from '@server/models/application/application' 10import { getOrCreateActorAndServerAndModel } from './actor'
11import { sendUndoAnnounce, sendVideoAnnounce } from './send'
12import { getLocalVideoAnnounceActivityPubUrl } from './url'
13
14const lTags = loggerTagsFactory('share')
13 15
14async function shareVideoByServerAndChannel (video: MVideoAccountLight, t: Transaction) { 16async function shareVideoByServerAndChannel (video: MVideoAccountLight, t: Transaction) {
15 if (!video.hasPrivacyForFederation()) return undefined 17 if (!video.hasPrivacyForFederation()) return undefined
@@ -25,7 +27,10 @@ async function changeVideoChannelShare (
25 oldVideoChannel: MChannelActorLight, 27 oldVideoChannel: MChannelActorLight,
26 t: Transaction 28 t: Transaction
27) { 29) {
28 logger.info('Updating video channel of video %s: %s -> %s.', video.uuid, oldVideoChannel.name, video.VideoChannel.name) 30 logger.info(
31 'Updating video channel of video %s: %s -> %s.', video.uuid, oldVideoChannel.name, video.VideoChannel.name,
32 lTags(video.uuid)
33 )
29 34
30 await undoShareByVideoChannel(video, oldVideoChannel, t) 35 await undoShareByVideoChannel(video, oldVideoChannel, t)
31 36
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts
index dbb37e0b2..37c43c3b0 100644
--- a/server/lib/video-blacklist.ts
+++ b/server/lib/video-blacklist.ts
@@ -11,7 +11,7 @@ import {
11} from '@server/types/models' 11} from '@server/types/models'
12import { UserRight, VideoBlacklistCreate, VideoBlacklistType } from '../../shared/models' 12import { UserRight, VideoBlacklistCreate, VideoBlacklistType } from '../../shared/models'
13import { UserAdminFlag } from '../../shared/models/users/user-flag.model' 13import { UserAdminFlag } from '../../shared/models/users/user-flag.model'
14import { logger } from '../helpers/logger' 14import { logger, loggerTagsFactory } from '../helpers/logger'
15import { CONFIG } from '../initializers/config' 15import { CONFIG } from '../initializers/config'
16import { VideoBlacklistModel } from '../models/video/video-blacklist' 16import { VideoBlacklistModel } from '../models/video/video-blacklist'
17import { sendDeleteVideo } from './activitypub/send' 17import { sendDeleteVideo } from './activitypub/send'
@@ -20,6 +20,8 @@ import { LiveManager } from './live-manager'
20import { Notifier } from './notifier' 20import { Notifier } from './notifier'
21import { Hooks } from './plugins/hooks' 21import { Hooks } from './plugins/hooks'
22 22
23const lTags = loggerTagsFactory('blacklist')
24
23async function autoBlacklistVideoIfNeeded (parameters: { 25async function autoBlacklistVideoIfNeeded (parameters: {
24 video: MVideoWithBlacklistLight 26 video: MVideoWithBlacklistLight
25 user?: MUser 27 user?: MUser
@@ -60,7 +62,7 @@ async function autoBlacklistVideoIfNeeded (parameters: {
60 }) 62 })
61 } 63 }
62 64
63 logger.info('Video %s auto-blacklisted.', video.uuid) 65 logger.info('Video %s auto-blacklisted.', video.uuid, lTags(video.uuid))
64 66
65 return true 67 return true
66} 68}