1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
import sys
import os
import simplejson as json
from datetime import datetime
from decimal import Decimal as D
import psycopg2
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from main import parse_config
config = sys.argv[1]
reports = sys.argv[2:]
pg_config, report_path = parse_config(config)
connection = psycopg2.connect(**pg_config)
cursor = connection.cursor()
report_query = 'INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id;'
line_query = 'INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);'
user_id_to_market_id = {
2: 1,
1: 3,
}
for report in reports:
with open(report, "rb") as f:
json_content = json.load(f, parse_float=D)
basename = os.path.basename(report)
date, rest = basename.split("_", 1)
user_id, rest = rest.split(".", 1)
date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
market_id = user_id_to_market_id[int(user_id)]
debug = any("debug" in x and x["debug"] for x in json_content)
print(market_id, date, debug)
cursor.execute(report_query, (date, market_id, debug))
report_id = cursor.fetchone()[0]
for line in json_content:
date = datetime.strptime(line["date"], "%Y-%m-%dT%H:%M:%S.%f")
type_ = line["type"]
del(line["date"])
del(line["type"])
cursor.execute(line_query, (date, report_id, type_, json.dumps(line)))
connection.commit()
cursor.close()
connection.close()
|
