Add watching and views endpoints to benchmark

[github/Chocobozzz/PeerTube.git] / scripts / parse-log.ts

diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts
index 81a22f7a0d3ba9b60d19e4c3ea50e322b72d157d..d3c93cee52c7852081d2e86066ef6f9fe5749637 100755 (executable)
--- a/scripts/parse-log.ts
+++ b/scripts/parse-log.ts
@@ -1,42 +1,155 @@
-import { createReadStream } from 'fs'
+import { program } from 'commander'
+import { createReadStream, readdir } from 'fs-extra'
 import { join } from 'path'
 import { createInterface } from 'readline'
+import { format as sqlFormat } from 'sql-formatter'
+import { inspect } from 'util'
 import * as winston from 'winston'
-import { readFileBufferPromise } from '../server/helpers/core-utils'
-import { CONFIG } from '../server/initializers/constants'
+import { labelFormatter, mtimeSortFilesDesc } from '../server/helpers/logger'
+import { CONFIG } from '../server/initializers/config'
 
-const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
+program
+  .option('-l, --level [level]', 'Level log (debug/info/warn/error)')
+  .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)')
+  .option('-t, --tags [tags...]', 'Display only lines with these tags')
+  .option('-nt, --not-tags [tags...]', 'Donrt display lines containing these tags')
+  .parse(process.argv)
 
-const logger = new winston.Logger({
+const options = program.opts()
+
+const excludedKeys = {
+  level: true,
+  message: true,
+  splat: true,
+  timestamp: true,
+  tags: true,
+  label: true,
+  sql: true
+}
+function keysExcluder (key, value) {
+  return excludedKeys[key] === true ? undefined : value
+}
+
+const loggerFormat = winston.format.printf((info) => {
+  let additionalInfos = JSON.stringify(info, keysExcluder, 2)
+  if (additionalInfos === '{}') additionalInfos = ''
+  else additionalInfos = ' ' + additionalInfos
+
+  if (info.sql) {
+    if (CONFIG.LOG.PRETTIFY_SQL) {
+      additionalInfos += '\n' + sqlFormat(info.sql, {
+        language: 'sql',
+        indent: '  '
+      })
+    } else {
+      additionalInfos += ' - ' + info.sql
+    }
+  }
+
+  return `[${info.label}] ${toTimeFormat(info.timestamp)} ${info.level}: ${info.message}${additionalInfos}`
+})
+
+const logger = winston.createLogger({
   transports: [
     new winston.transports.Console({
-      level: 'debug',
-      label: label,
-      handleExceptions: true,
-      humanReadableUnhandledException: true,
-      json: false,
-      colorize: true,
-      prettyPrint: true
+      level: options.level || 'debug',
+      stderrLevels: [],
+      format: winston.format.combine(
+        winston.format.splat(),
+        labelFormatter(),
+        winston.format.colorize(),
+        loggerFormat
+      )
     })
   ],
   exitOnError: true
 })
 
 const logLevels = {
-  error: logger.error,
-  warn: logger.warn,
-  info: logger.info,
-  debug: logger.debug
+  error: logger.error.bind(logger),
+  warn: logger.warn.bind(logger),
+  info: logger.info.bind(logger),
+  debug: logger.debug.bind(logger)
 }
 
-const path = join(CONFIG.STORAGE.LOG_DIR, 'all-logs.log')
-console.log('Opening %s.', path)
+run()
+  .then(() => process.exit(0))
+  .catch(err => console.error(err))
 
-const rl = createInterface({
-  input: createReadStream(path)
-})
+function run () {
+  return new Promise<void>(async res => {
+    const files = await getFiles()
 
-rl.on('line', line => {
-  const log = JSON.parse(line)
-  logLevels[log.level](log.message)
-})
+    for (const file of files) {
+      if (file === 'peertube-audit.log') continue
+
+      console.log('Opening %s.', file)
+
+      const stream = createReadStream(file)
+
+      const rl = createInterface({
+        input: stream
+      })
+
+      rl.on('line', line => {
+        try {
+          const log = JSON.parse(line)
+          if (options.tags && !containsTags(log.tags, options.tags)) {
+            return
+          }
+
+          if (options.notTags && containsTags(log.tags, options.notTags)) {
+            return
+          }
+
+          // Don't know why but loggerFormat does not remove splat key
+          Object.assign(log, { splat: undefined })
+
+          logLevels[log.level](log)
+        } catch (err) {
+          console.error('Cannot parse line.', inspect(line))
+          throw err
+        }
+      })
+
+      stream.once('close', () => res())
+    }
+  })
+}
+
+// Thanks: https://stackoverflow.com/a/37014317
+async function getNewestFile (files: string[], basePath: string) {
+  const sorted = await mtimeSortFilesDesc(files, basePath)
+
+  return (sorted.length > 0) ? sorted[0].file : ''
+}
+
+async function getFiles () {
+  if (options.files) return options.files
+
+  const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
+
+  const filename = await getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR)
+  return [ join(CONFIG.STORAGE.LOG_DIR, filename) ]
+}
+
+function toTimeFormat (time: string) {
+  const timestamp = Date.parse(time)
+
+  if (isNaN(timestamp) === true) return 'Unknown date'
+
+  const d = new Date(timestamp)
+  return d.toLocaleString() + `.${d.getMilliseconds()}`
+}
+
+function containsTags (loggerTags: string[], optionsTags: string[]) {
+  if (!loggerTags) return false
+
+  for (const lt of loggerTags) {
+    for (const ot of optionsTags) {
+      if (lt === ot) return true
+    }
+  }
+
+  return false
+}