scripts/parse-log.ts

   1 import { program } from 'commander'
   2 import { createReadStream, readdir } from 'fs-extra'
   3 import { join } from 'path'
   4 import { stdin } from 'process'
   5 import { createInterface } from 'readline'
   6 import { format as sqlFormat } from 'sql-formatter'
   7 import { inspect } from 'util'
   8 import * as winston from 'winston'
   9 import { labelFormatter, mtimeSortFilesDesc } from '../server/helpers/logger'
  10 import { CONFIG } from '../server/initializers/config'
  11
  12 program
  13   .option('-l, --level [level]', 'Level log (debug/info/warn/error)')
  14   .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)')
  15   .option('-t, --tags [tags...]', 'Display only lines with these tags')
  16   .option('-nt, --not-tags [tags...]', 'Donrt display lines containing these tags')
  17   .parse(process.argv)
  18
  19 const options = program.opts()
  20
  21 const excludedKeys = {
  22   level: true,
  23   message: true,
  24   splat: true,
  25   timestamp: true,
  26   tags: true,
  27   label: true,
  28   sql: true
  29 }
  30 function keysExcluder (key, value) {
  31   return excludedKeys[key] === true ? undefined : value
  32 }
  33
  34 const loggerFormat = winston.format.printf((info) => {
  35   let additionalInfos = JSON.stringify(info, keysExcluder, 2)
  36   if (additionalInfos === '{}') additionalInfos = ''
  37   else additionalInfos = ' ' + additionalInfos
  38
  39   if (info.sql) {
  40     if (CONFIG.LOG.PRETTIFY_SQL) {
  41       additionalInfos += '\n' + sqlFormat(info.sql, {
  42         language: 'sql',
  43         tabWidth: 2
  44       })
  45     } else {
  46       additionalInfos += ' - ' + info.sql
  47     }
  48   }
  49
  50   return `[${info.label}] ${toTimeFormat(info.timestamp)} ${info.level}: ${info.message}${additionalInfos}`
  51 })
  52
  53 const logger = winston.createLogger({
  54   transports: [
  55     new winston.transports.Console({
  56       level: options.level || 'debug',
  57       stderrLevels: [],
  58       format: winston.format.combine(
  59         winston.format.splat(),
  60         labelFormatter(),
  61         winston.format.colorize(),
  62         loggerFormat
  63       )
  64     })
  65   ],
  66   exitOnError: true
  67 })
  68
  69 const logLevels = {
  70   error: logger.error.bind(logger),
  71   warn: logger.warn.bind(logger),
  72   info: logger.info.bind(logger),
  73   debug: logger.debug.bind(logger)
  74 }
  75
  76 run()
  77   .then(() => process.exit(0))
  78   .catch(err => console.error(err))
  79
  80 async function run () {
  81   const files = await getFiles()
  82
  83   for (const file of files) {
  84     if (file === 'peertube-audit.log') continue
  85
  86     await readFile(file)
  87   }
  88 }
  89
  90 function readFile (file: string) {
  91   console.log('Opening %s.', file)
  92
  93   const stream = file === '-' ? stdin : createReadStream(file)
  94
  95   const rl = createInterface({
  96     input: stream
  97   })
  98
  99   return new Promise<void>(res => {
 100     rl.on('line', line => {
 101       try {
 102         const log = JSON.parse(line)
 103         if (options.tags && !containsTags(log.tags, options.tags)) {
 104           return
 105         }
 106
 107         if (options.notTags && containsTags(log.tags, options.notTags)) {
 108           return
 109         }
 110
 111         // Don't know why but loggerFormat does not remove splat key
 112         Object.assign(log, { splat: undefined })
 113
 114         logLevels[log.level](log)
 115       } catch (err) {
 116         console.error('Cannot parse line.', inspect(line))
 117         throw err
 118       }
 119     })
 120
 121     stream.once('end', () => res())
 122   })
 123 }
 124
 125 // Thanks: https://stackoverflow.com/a/37014317
 126 async function getNewestFile (files: string[], basePath: string) {
 127   const sorted = await mtimeSortFilesDesc(files, basePath)
 128
 129   return (sorted.length > 0) ? sorted[0].file : ''
 130 }
 131
 132 async function getFiles () {
 133   if (options.files) return options.files
 134
 135   const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
 136
 137   const filename = await getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR)
 138   return [ join(CONFIG.STORAGE.LOG_DIR, filename) ]
 139 }
 140
 141 function toTimeFormat (time: string) {
 142   const timestamp = Date.parse(time)
 143
 144   if (isNaN(timestamp) === true) return 'Unknown date'
 145
 146   const d = new Date(timestamp)
 147   return d.toLocaleString() + `.${d.getMilliseconds()}`
 148 }
 149
 150 function containsTags (loggerTags: string[], optionsTags: string[]) {
 151   if (!loggerTags) return false
 152
 153   for (const lt of loggerTags) {
 154     for (const ot of optionsTags) {
 155       if (lt === ot) return true
 156     }
 157   }
 158
 159   return false
 160 }