diff options
-rw-r--r-- | tasks/import_reports_to_database.py | 15 |
1 files changed, 6 insertions, 9 deletions
diff --git a/tasks/import_reports_to_database.py b/tasks/import_reports_to_database.py index 152c762..6031cbe 100644 --- a/tasks/import_reports_to_database.py +++ b/tasks/import_reports_to_database.py | |||
@@ -16,13 +16,9 @@ pg_config, report_path = parse_config(config) | |||
16 | connection = psycopg2.connect(**pg_config) | 16 | connection = psycopg2.connect(**pg_config) |
17 | cursor = connection.cursor() | 17 | cursor = connection.cursor() |
18 | 18 | ||
19 | report_query = 'INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id;' | 19 | report_query = 'INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id' |
20 | line_query = 'INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);' | 20 | line_query = 'INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s)' |
21 | 21 | market_config_query = "SELECT id FROM market_configs WHERE user_id = %s AND market_name = 'poloniex'" | |
22 | user_id_to_market_id = { | ||
23 | 2: 1, | ||
24 | 1: 3, | ||
25 | } | ||
26 | 22 | ||
27 | for report in reports: | 23 | for report in reports: |
28 | with open(report, "rb") as f: | 24 | with open(report, "rb") as f: |
@@ -32,7 +28,8 @@ for report in reports: | |||
32 | user_id, rest = rest.split(".", 1) | 28 | user_id, rest = rest.split(".", 1) |
33 | 29 | ||
34 | date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") | 30 | date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") |
35 | market_id = user_id_to_market_id[int(user_id)] | 31 | cursor.execute(market_config_query, user_id) |
32 | market_id = cursor.fetchone()[0] | ||
36 | debug = any("debug" in x and x["debug"] for x in json_content) | 33 | debug = any("debug" in x and x["debug"] for x in json_content) |
37 | print(market_id, date, debug) | 34 | print(market_id, date, debug) |
38 | cursor.execute(report_query, (date, market_id, debug)) | 35 | cursor.execute(report_query, (date, market_id, debug)) |
@@ -44,7 +41,7 @@ for report in reports: | |||
44 | del(line["date"]) | 41 | del(line["date"]) |
45 | del(line["type"]) | 42 | del(line["type"]) |
46 | 43 | ||
47 | cursor.execute(line_query, (date, report_id, type_, json.dumps(line))) | 44 | cursor.execute(line_query, (date, report_id, type_, json.dumps(line, indent=" "))) |
48 | connection.commit() | 45 | connection.commit() |
49 | cursor.close() | 46 | cursor.close() |
50 | connection.close() | 47 | connection.close() |