scripts/parse-log.ts

   1 import { program } from 'commander'
   2 import { createReadStream, readdir } from 'fs-extra'
   3 import { join } from 'path'
   4 import { createInterface } from 'readline'
   5 import { format as sqlFormat } from 'sql-formatter'
   6 import { inspect } from 'util'
   7 import * as winston from 'winston'
   8 import { labelFormatter, mtimeSortFilesDesc } from '../server/helpers/logger'
   9 import { CONFIG } from '../server/initializers/config'
  10
  11 program
  12   .option('-l, --level [level]', 'Level log (debug/info/warn/error)')
  13   .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)')
  14   .option('-t, --tags [tags...]', 'Display only lines with these tags')
  15   .option('-nt, --not-tags [tags...]', 'Donrt display lines containing these tags')
  16   .parse(process.argv)
  17
  18 const options = program.opts()
  19
  20 const excludedKeys = {
  21   level: true,
  22   message: true,
  23   splat: true,
  24   timestamp: true,
  25   tags: true,
  26   label: true,
  27   sql: true
  28 }
  29 function keysExcluder (key, value) {
  30   return excludedKeys[key] === true ? undefined : value
  31 }
  32
  33 const loggerFormat = winston.format.printf((info) => {
  34   let additionalInfos = JSON.stringify(info, keysExcluder, 2)
  35   if (additionalInfos === '{}') additionalInfos = ''
  36   else additionalInfos = ' ' + additionalInfos
  37
  38   if (info.sql) {
  39     if (CONFIG.LOG.PRETTIFY_SQL) {
  40       additionalInfos += '\n' + sqlFormat(info.sql, {
  41         language: 'sql',
  42         indent: '  '
  43       })
  44     } else {
  45       additionalInfos += ' - ' + info.sql
  46     }
  47   }
  48
  49   return `[${info.label}] ${toTimeFormat(info.timestamp)} ${info.level}: ${info.message}${additionalInfos}`
  50 })
  51
  52 const logger = winston.createLogger({
  53   transports: [
  54     new winston.transports.Console({
  55       level: options.level || 'debug',
  56       stderrLevels: [],
  57       format: winston.format.combine(
  58         winston.format.splat(),
  59         labelFormatter(),
  60         winston.format.colorize(),
  61         loggerFormat
  62       )
  63     })
  64   ],
  65   exitOnError: true
  66 })
  67
  68 const logLevels = {
  69   error: logger.error.bind(logger),
  70   warn: logger.warn.bind(logger),
  71   info: logger.info.bind(logger),
  72   debug: logger.debug.bind(logger)
  73 }
  74
  75 run()
  76   .then(() => process.exit(0))
  77   .catch(err => console.error(err))
  78
  79 function run () {
  80   return new Promise<void>(async res => {
  81     const files = await getFiles()
  82
  83     for (const file of files) {
  84       if (file === 'peertube-audit.log') continue
  85
  86       console.log('Opening %s.', file)
  87
  88       const stream = createReadStream(file)
  89
  90       const rl = createInterface({
  91         input: stream
  92       })
  93
  94       rl.on('line', line => {
  95         try {
  96           const log = JSON.parse(line)
  97           if (options.tags && !containsTags(log.tags, options.tags)) {
  98             return
  99           }
 100
 101           if (options.notTags && containsTags(log.tags, options.notTags)) {
 102             return
 103           }
 104
 105           // Don't know why but loggerFormat does not remove splat key
 106           Object.assign(log, { splat: undefined })
 107
 108           logLevels[log.level](log)
 109         } catch (err) {
 110           console.error('Cannot parse line.', inspect(line))
 111           throw err
 112         }
 113       })
 114
 115       stream.once('close', () => res())
 116     }
 117   })
 118 }
 119
 120 // Thanks: https://stackoverflow.com/a/37014317
 121 async function getNewestFile (files: string[], basePath: string) {
 122   const sorted = await mtimeSortFilesDesc(files, basePath)
 123
 124   return (sorted.length > 0) ? sorted[0].file : ''
 125 }
 126
 127 async function getFiles () {
 128   if (options.files) return options.files
 129
 130   const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
 131
 132   const filename = await getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR)
 133   return [ join(CONFIG.STORAGE.LOG_DIR, filename) ]
 134 }
 135
 136 function toTimeFormat (time: string) {
 137   const timestamp = Date.parse(time)
 138
 139   if (isNaN(timestamp) === true) return 'Unknown date'
 140
 141   const d = new Date(timestamp)
 142   return d.toLocaleString() + `.${d.getMilliseconds()}`
 143 }
 144
 145 function containsTags (loggerTags: string[], optionsTags: string[]) {
 146   if (!loggerTags) return false
 147
 148   for (const lt of loggerTags) {
 149     for (const ot of optionsTags) {
 150       if (lt === ot) return true
 151     }
 152   }
 153
 154   return false
 155 }