aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--main.py30
-rw-r--r--market.py51
-rw-r--r--store.py20
-rw-r--r--tasks/import_reports_to_database.py50
-rw-r--r--test.py212
5 files changed, 286 insertions, 77 deletions
diff --git a/main.py b/main.py
index 856d449..4462192 100644
--- a/main.py
+++ b/main.py
@@ -62,17 +62,20 @@ def make_order(market, value, currency, action="acquire",
62 62
63def get_user_market(config_path, user_id, debug=False): 63def get_user_market(config_path, user_id, debug=False):
64 pg_config, report_path = parse_config(config_path) 64 pg_config, report_path = parse_config(config_path)
65 market_config = list(fetch_markets(pg_config, str(user_id)))[0][0] 65 market_id, market_config, user_id = list(fetch_markets(pg_config, str(user_id)))[0]
66 return market.Market.from_config(market_config, debug=debug) 66 args = type('Args', (object,), { "debug": debug, "quiet": False })()
67 return market.Market.from_config(market_config, args,
68 pg_config=pg_config, market_id=market_id,
69 user_id=user_id, report_path=report_path)
67 70
68def fetch_markets(pg_config, user): 71def fetch_markets(pg_config, user):
69 connection = psycopg2.connect(**pg_config) 72 connection = psycopg2.connect(**pg_config)
70 cursor = connection.cursor() 73 cursor = connection.cursor()
71 74
72 if user is None: 75 if user is None:
73 cursor.execute("SELECT config,user_id FROM market_configs") 76 cursor.execute("SELECT id,config,user_id FROM market_configs")
74 else: 77 else:
75 cursor.execute("SELECT config,user_id FROM market_configs WHERE user_id = %s", user) 78 cursor.execute("SELECT id,config,user_id FROM market_configs WHERE user_id = %s", user)
76 79
77 for row in cursor: 80 for row in cursor:
78 yield row 81 yield row
@@ -109,6 +112,9 @@ def parse_args(argv):
109 parser.add_argument("--after", 112 parser.add_argument("--after",
110 default=False, action='store_const', const=True, 113 default=False, action='store_const', const=True,
111 help="Run the steps after the cryptoportfolio update") 114 help="Run the steps after the cryptoportfolio update")
115 parser.add_argument("--quiet",
116 default=False, action='store_const', const=True,
117 help="Don't print messages")
112 parser.add_argument("--debug", 118 parser.add_argument("--debug",
113 default=False, action='store_const', const=True, 119 default=False, action='store_const', const=True,
114 help="Run in debug mode") 120 help="Run in debug mode")
@@ -128,10 +134,12 @@ def parse_args(argv):
128 134
129 return args 135 return args
130 136
131def process(market_config, user_id, report_path, args): 137def process(market_config, market_id, user_id, args, report_path, pg_config):
132 try: 138 try:
133 market.Market\ 139 market.Market\
134 .from_config(market_config, debug=args.debug, user_id=user_id, report_path=report_path)\ 140 .from_config(market_config, args,
141 pg_config=pg_config, market_id=market_id,
142 user_id=user_id, report_path=report_path)\
135 .process(args.action, before=args.before, after=args.after) 143 .process(args.action, before=args.before, after=args.after)
136 except Exception as e: 144 except Exception as e:
137 print("{}: {}".format(e.__class__.__name__, e)) 145 print("{}: {}".format(e.__class__.__name__, e))
@@ -145,11 +153,13 @@ def main(argv):
145 import threading 153 import threading
146 market.Portfolio.start_worker() 154 market.Portfolio.start_worker()
147 155
148 for market_config, user_id in fetch_markets(pg_config, args.user): 156 def process_(*args):
149 threading.Thread(target=process, args=[market_config, user_id, report_path, args]).start() 157 threading.Thread(target=process, args=args).start()
150 else: 158 else:
151 for market_config, user_id in fetch_markets(pg_config, args.user): 159 process_ = process
152 process(market_config, user_id, report_path, args) 160
161 for market_id, market_config, user_id in fetch_markets(pg_config, args.user):
162 process_(market_config, market_id, user_id, args, report_path, pg_config)
153 163
154if __name__ == '__main__': # pragma: no cover 164if __name__ == '__main__': # pragma: no cover
155 main(sys.argv[1:]) 165 main(sys.argv[1:])
diff --git a/market.py b/market.py
index 2ddebfa..496ec45 100644
--- a/market.py
+++ b/market.py
@@ -1,6 +1,7 @@
1from ccxt import ExchangeError, NotSupported 1from ccxt import ExchangeError, NotSupported
2import ccxt_wrapper as ccxt 2import ccxt_wrapper as ccxt
3import time 3import time
4import psycopg2
4from store import * 5from store import *
5from cachetools.func import ttl_cache 6from cachetools.func import ttl_cache
6from datetime import datetime 7from datetime import datetime
@@ -13,20 +14,21 @@ class Market:
13 trades = None 14 trades = None
14 balances = None 15 balances = None
15 16
16 def __init__(self, ccxt_instance, debug=False, user_id=None, report_path=None): 17 def __init__(self, ccxt_instance, args, **kwargs):
17 self.debug = debug 18 self.args = args
19 self.debug = args.debug
18 self.ccxt = ccxt_instance 20 self.ccxt = ccxt_instance
19 self.ccxt._market = self 21 self.ccxt._market = self
20 self.report = ReportStore(self) 22 self.report = ReportStore(self, verbose_print=(not args.quiet))
21 self.trades = TradeStore(self) 23 self.trades = TradeStore(self)
22 self.balances = BalanceStore(self) 24 self.balances = BalanceStore(self)
23 self.processor = Processor(self) 25 self.processor = Processor(self)
24 26
25 self.user_id = user_id 27 for key in ["user_id", "market_id", "report_path", "pg_config"]:
26 self.report_path = report_path 28 setattr(self, key, kwargs.get(key, None))
27 29
28 @classmethod 30 @classmethod
29 def from_config(cls, config, debug=False, user_id=None, report_path=None): 31 def from_config(cls, config, args, **kwargs):
30 config["apiKey"] = config.pop("key", None) 32 config["apiKey"] = config.pop("key", None)
31 33
32 ccxt_instance = ccxt.poloniexE(config) 34 ccxt_instance = ccxt.poloniexE(config)
@@ -43,20 +45,43 @@ class Market:
43 ccxt_instance.session.request = request_wrap.__get__(ccxt_instance.session, 45 ccxt_instance.session.request = request_wrap.__get__(ccxt_instance.session,
44 ccxt_instance.session.__class__) 46 ccxt_instance.session.__class__)
45 47
46 return cls(ccxt_instance, debug=debug, user_id=user_id, report_path=report_path) 48 return cls(ccxt_instance, args, **kwargs)
47 49
48 def store_report(self): 50 def store_report(self):
49 self.report.merge(Portfolio.report) 51 self.report.merge(Portfolio.report)
52 date = datetime.now()
53 if self.report_path is not None:
54 self.store_file_report(date)
55 if self.pg_config is not None:
56 self.store_database_report(date)
57
58 def store_file_report(self, date):
50 try: 59 try:
51 if self.report_path is not None: 60 report_file = "{}/{}_{}".format(self.report_path, date.isoformat(), self.user_id)
52 report_file = "{}/{}_{}".format(self.report_path, datetime.now().isoformat(), self.user_id) 61 with open(report_file + ".json", "w") as f:
53 with open(report_file + ".json", "w") as f: 62 f.write(self.report.to_json())
54 f.write(self.report.to_json()) 63 with open(report_file + ".log", "w") as f:
55 with open(report_file + ".log", "w") as f: 64 f.write("\n".join(map(lambda x: x[1], self.report.print_logs)))
56 f.write("\n".join(map(lambda x: x[1], self.report.print_logs)))
57 except Exception as e: 65 except Exception as e:
58 print("impossible to store report file: {}; {}".format(e.__class__.__name__, e)) 66 print("impossible to store report file: {}; {}".format(e.__class__.__name__, e))
59 67
68 def store_database_report(self, date):
69 try:
70 report_query = 'INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id;'
71 line_query = 'INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);'
72 connection = psycopg2.connect(**self.pg_config)
73 cursor = connection.cursor()
74 cursor.execute(report_query, (date, self.market_id, self.debug))
75 report_id = cursor.fetchone()[0]
76 for date, type_, payload in self.report.to_json_array():
77 cursor.execute(line_query, (date, report_id, type_, payload))
78
79 connection.commit()
80 cursor.close()
81 connection.close()
82 except Exception as e:
83 print("impossible to store report to database: {}; {}".format(e.__class__.__name__, e))
84
60 def process(self, actions, before=False, after=False): 85 def process(self, actions, before=False, after=False):
61 try: 86 try:
62 if len(actions or []) == 0: 87 if len(actions or []) == 0:
diff --git a/store.py b/store.py
index d875a98..b3ada45 100644
--- a/store.py
+++ b/store.py
@@ -36,12 +36,22 @@ class ReportStore:
36 hash_["date"] = datetime.now() 36 hash_["date"] = datetime.now()
37 self.logs.append(hash_) 37 self.logs.append(hash_)
38 38
39 @staticmethod
40 def default_json_serial(obj):
41 if isinstance(obj, (datetime, date)):
42 return obj.isoformat()
43 return str(obj)
44
39 def to_json(self): 45 def to_json(self):
40 def default_json_serial(obj): 46 return json.dumps(self.logs, default=self.default_json_serial, indent=" ")
41 if isinstance(obj, (datetime, date)): 47
42 return obj.isoformat() 48 def to_json_array(self):
43 return str(obj) 49 for log in (x.copy() for x in self.logs):
44 return json.dumps(self.logs, default=default_json_serial, indent=" ") 50 yield (
51 log.pop("date"),
52 log.pop("type"),
53 json.dumps(log, default=self.default_json_serial, indent=" ")
54 )
45 55
46 def set_verbose(self, verbose_print): 56 def set_verbose(self, verbose_print):
47 self.verbose_print = verbose_print 57 self.verbose_print = verbose_print
diff --git a/tasks/import_reports_to_database.py b/tasks/import_reports_to_database.py
new file mode 100644
index 0000000..152c762
--- /dev/null
+++ b/tasks/import_reports_to_database.py
@@ -0,0 +1,50 @@
1import sys
2import os
3import simplejson as json
4from datetime import datetime
5from decimal import Decimal as D
6import psycopg2
7
8sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
9from main import parse_config
10
11config = sys.argv[1]
12reports = sys.argv[2:]
13
14pg_config, report_path = parse_config(config)
15
16connection = psycopg2.connect(**pg_config)
17cursor = connection.cursor()
18
19report_query = 'INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id;'
20line_query = 'INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);'
21
22user_id_to_market_id = {
23 2: 1,
24 1: 3,
25 }
26
27for report in reports:
28 with open(report, "rb") as f:
29 json_content = json.load(f, parse_float=D)
30 basename = os.path.basename(report)
31 date, rest = basename.split("_", 1)
32 user_id, rest = rest.split(".", 1)
33
34 date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
35 market_id = user_id_to_market_id[int(user_id)]
36 debug = any("debug" in x and x["debug"] for x in json_content)
37 print(market_id, date, debug)
38 cursor.execute(report_query, (date, market_id, debug))
39 report_id = cursor.fetchone()[0]
40
41 for line in json_content:
42 date = datetime.strptime(line["date"], "%Y-%m-%dT%H:%M:%S.%f")
43 type_ = line["type"]
44 del(line["date"])
45 del(line["type"])
46
47 cursor.execute(line_query, (date, report_id, type_, json.dumps(line)))
48connection.commit()
49cursor.close()
50connection.close()
diff --git a/test.py b/test.py
index 13bd332..637a305 100644
--- a/test.py
+++ b/test.py
@@ -23,6 +23,9 @@ for test_type in limits:
23class WebMockTestCase(unittest.TestCase): 23class WebMockTestCase(unittest.TestCase):
24 import time 24 import time
25 25
26 def market_args(self, debug=False, quiet=False):
27 return type('Args', (object,), { "debug": debug, "quiet": quiet })()
28
26 def setUp(self): 29 def setUp(self):
27 super(WebMockTestCase, self).setUp() 30 super(WebMockTestCase, self).setUp()
28 self.wm = requests_mock.Mocker() 31 self.wm = requests_mock.Mocker()
@@ -1092,7 +1095,7 @@ class MarketTest(WebMockTestCase):
1092 self.ccxt = mock.Mock(spec=market.ccxt.poloniexE) 1095 self.ccxt = mock.Mock(spec=market.ccxt.poloniexE)
1093 1096
1094 def test_values(self): 1097 def test_values(self):
1095 m = market.Market(self.ccxt) 1098 m = market.Market(self.ccxt, self.market_args())
1096 1099
1097 self.assertEqual(self.ccxt, m.ccxt) 1100 self.assertEqual(self.ccxt, m.ccxt)
1098 self.assertFalse(m.debug) 1101 self.assertFalse(m.debug)
@@ -1104,19 +1107,27 @@ class MarketTest(WebMockTestCase):
1104 self.assertEqual(m, m.balances.market) 1107 self.assertEqual(m, m.balances.market)
1105 self.assertEqual(m, m.ccxt._market) 1108 self.assertEqual(m, m.ccxt._market)
1106 1109
1107 m = market.Market(self.ccxt, debug=True) 1110 m = market.Market(self.ccxt, self.market_args(debug=True))
1108 self.assertTrue(m.debug) 1111 self.assertTrue(m.debug)
1109 1112
1110 m = market.Market(self.ccxt, debug=False) 1113 m = market.Market(self.ccxt, self.market_args(debug=False))
1111 self.assertFalse(m.debug) 1114 self.assertFalse(m.debug)
1112 1115
1116 with mock.patch("market.ReportStore") as report_store:
1117 with self.subTest(quiet=False):
1118 m = market.Market(self.ccxt, self.market_args(quiet=False))
1119 report_store.assert_called_with(m, verbose_print=True)
1120 with self.subTest(quiet=True):
1121 m = market.Market(self.ccxt, self.market_args(quiet=True))
1122 report_store.assert_called_with(m, verbose_print=False)
1123
1113 @mock.patch("market.ccxt") 1124 @mock.patch("market.ccxt")
1114 def test_from_config(self, ccxt): 1125 def test_from_config(self, ccxt):
1115 with mock.patch("market.ReportStore"): 1126 with mock.patch("market.ReportStore"):
1116 ccxt.poloniexE.return_value = self.ccxt 1127 ccxt.poloniexE.return_value = self.ccxt
1117 self.ccxt.session.request.return_value = "response" 1128 self.ccxt.session.request.return_value = "response"
1118 1129
1119 m = market.Market.from_config({"key": "key", "secred": "secret"}) 1130 m = market.Market.from_config({"key": "key", "secred": "secret"}, self.market_args())
1120 1131
1121 self.assertEqual(self.ccxt, m.ccxt) 1132 self.assertEqual(self.ccxt, m.ccxt)
1122 1133
@@ -1125,7 +1136,7 @@ class MarketTest(WebMockTestCase):
1125 m.report.log_http_request.assert_called_with('GET', 'URL', 'data', 1136 m.report.log_http_request.assert_called_with('GET', 'URL', 'data',
1126 'headers', 'response') 1137 'headers', 'response')
1127 1138
1128 m = market.Market.from_config({"key": "key", "secred": "secret"}, debug=True) 1139 m = market.Market.from_config({"key": "key", "secred": "secret"}, self.market_args(debug=True))
1129 self.assertEqual(True, m.debug) 1140 self.assertEqual(True, m.debug)
1130 1141
1131 def test_get_tickers(self): 1142 def test_get_tickers(self):
@@ -1134,7 +1145,7 @@ class MarketTest(WebMockTestCase):
1134 market.NotSupported 1145 market.NotSupported
1135 ] 1146 ]
1136 1147
1137 m = market.Market(self.ccxt) 1148 m = market.Market(self.ccxt, self.market_args())
1138 self.assertEqual("tickers", m.get_tickers()) 1149 self.assertEqual("tickers", m.get_tickers())
1139 self.assertEqual("tickers", m.get_tickers()) 1150 self.assertEqual("tickers", m.get_tickers())
1140 self.ccxt.fetch_tickers.assert_called_once() 1151 self.ccxt.fetch_tickers.assert_called_once()
@@ -1147,7 +1158,7 @@ class MarketTest(WebMockTestCase):
1147 "ETH/ETC": { "bid": 1, "ask": 3 }, 1158 "ETH/ETC": { "bid": 1, "ask": 3 },
1148 "XVG/ETH": { "bid": 10, "ask": 40 }, 1159 "XVG/ETH": { "bid": 10, "ask": 40 },
1149 } 1160 }
1150 m = market.Market(self.ccxt) 1161 m = market.Market(self.ccxt, self.market_args())
1151 1162
1152 ticker = m.get_ticker("ETH", "ETC") 1163 ticker = m.get_ticker("ETH", "ETC")
1153 self.assertEqual(1, ticker["bid"]) 1164 self.assertEqual(1, ticker["bid"])
@@ -1175,7 +1186,7 @@ class MarketTest(WebMockTestCase):
1175 market.ExchangeError("foo"), 1186 market.ExchangeError("foo"),
1176 ] 1187 ]
1177 1188
1178 m = market.Market(self.ccxt) 1189 m = market.Market(self.ccxt, self.market_args())
1179 1190
1180 ticker = m.get_ticker("ETH", "ETC") 1191 ticker = m.get_ticker("ETH", "ETC")
1181 self.ccxt.fetch_ticker.assert_called_with("ETH/ETC") 1192 self.ccxt.fetch_ticker.assert_called_with("ETH/ETC")
@@ -1195,7 +1206,7 @@ class MarketTest(WebMockTestCase):
1195 self.assertIsNone(ticker) 1206 self.assertIsNone(ticker)
1196 1207
1197 def test_fetch_fees(self): 1208 def test_fetch_fees(self):
1198 m = market.Market(self.ccxt) 1209 m = market.Market(self.ccxt, self.market_args())
1199 self.ccxt.fetch_fees.return_value = "Foo" 1210 self.ccxt.fetch_fees.return_value = "Foo"
1200 self.assertEqual("Foo", m.fetch_fees()) 1211 self.assertEqual("Foo", m.fetch_fees())
1201 self.ccxt.fetch_fees.assert_called_once() 1212 self.ccxt.fetch_fees.assert_called_once()
@@ -1222,7 +1233,7 @@ class MarketTest(WebMockTestCase):
1222 get_ticker.side_effect = _get_ticker 1233 get_ticker.side_effect = _get_ticker
1223 1234
1224 with mock.patch("market.ReportStore"): 1235 with mock.patch("market.ReportStore"):
1225 m = market.Market(self.ccxt) 1236 m = market.Market(self.ccxt, self.market_args())
1226 self.ccxt.fetch_all_balances.return_value = { 1237 self.ccxt.fetch_all_balances.return_value = {
1227 "USDT": { 1238 "USDT": {
1228 "exchange_free": D("10000.0"), 1239 "exchange_free": D("10000.0"),
@@ -1262,7 +1273,7 @@ class MarketTest(WebMockTestCase):
1262 (False, 12), (True, 12)]: 1273 (False, 12), (True, 12)]:
1263 with self.subTest(sleep=sleep, debug=debug), \ 1274 with self.subTest(sleep=sleep, debug=debug), \
1264 mock.patch("market.ReportStore"): 1275 mock.patch("market.ReportStore"):
1265 m = market.Market(self.ccxt, debug=debug) 1276 m = market.Market(self.ccxt, self.market_args(debug=debug))
1266 1277
1267 order_mock1 = mock.Mock() 1278 order_mock1 = mock.Mock()
1268 order_mock2 = mock.Mock() 1279 order_mock2 = mock.Mock()
@@ -1339,7 +1350,7 @@ class MarketTest(WebMockTestCase):
1339 for debug in [True, False]: 1350 for debug in [True, False]:
1340 with self.subTest(debug=debug),\ 1351 with self.subTest(debug=debug),\
1341 mock.patch("market.ReportStore"): 1352 mock.patch("market.ReportStore"):
1342 m = market.Market(self.ccxt, debug=debug) 1353 m = market.Market(self.ccxt, self.market_args(debug=debug))
1343 1354
1344 value_from = portfolio.Amount("BTC", "1.0") 1355 value_from = portfolio.Amount("BTC", "1.0")
1345 value_from.linked_to = portfolio.Amount("ETH", "10.0") 1356 value_from.linked_to = portfolio.Amount("ETH", "10.0")
@@ -1375,54 +1386,135 @@ class MarketTest(WebMockTestCase):
1375 self.ccxt.transfer_balance.assert_any_call("USDT", 100, "exchange", "margin") 1386 self.ccxt.transfer_balance.assert_any_call("USDT", 100, "exchange", "margin")
1376 self.ccxt.transfer_balance.assert_any_call("ETC", 5, "margin", "exchange") 1387 self.ccxt.transfer_balance.assert_any_call("ETC", 5, "margin", "exchange")
1377 1388
1378 def test_store_report(self): 1389 def test_store_file_report(self):
1379
1380 file_open = mock.mock_open()
1381 m = market.Market(self.ccxt, user_id=1)
1382 with self.subTest(file=None),\
1383 mock.patch.object(m, "report") as report,\
1384 mock.patch("market.open", file_open):
1385 m.store_report()
1386 report.merge.assert_called_with(store.Portfolio.report)
1387 file_open.assert_not_called()
1388
1389 report.reset_mock()
1390 file_open = mock.mock_open() 1390 file_open = mock.mock_open()
1391 m = market.Market(self.ccxt, report_path="present", user_id=1) 1391 m = market.Market(self.ccxt, self.market_args(), report_path="present", user_id=1)
1392 with self.subTest(file="present"),\ 1392 with self.subTest(file="present"),\
1393 mock.patch("market.open", file_open),\ 1393 mock.patch("market.open", file_open),\
1394 mock.patch.object(m, "report") as report,\ 1394 mock.patch.object(m, "report") as report,\
1395 mock.patch.object(market, "datetime") as time_mock: 1395 mock.patch.object(market, "datetime") as time_mock:
1396 1396
1397 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
1398 report.print_logs = [[time_mock.now(), "Foo"], [time_mock.now(), "Bar"]] 1397 report.print_logs = [[time_mock.now(), "Foo"], [time_mock.now(), "Bar"]]
1399 report.to_json.return_value = "json_content" 1398 report.to_json.return_value = "json_content"
1400 1399
1401 m.store_report() 1400 m.store_file_report(datetime.datetime(2018, 2, 25))
1402 1401
1403 file_open.assert_any_call("present/2018-02-25T00:00:00_1.json", "w") 1402 file_open.assert_any_call("present/2018-02-25T00:00:00_1.json", "w")
1404 file_open.assert_any_call("present/2018-02-25T00:00:00_1.log", "w") 1403 file_open.assert_any_call("present/2018-02-25T00:00:00_1.log", "w")
1405 file_open().write.assert_any_call("json_content") 1404 file_open().write.assert_any_call("json_content")
1406 file_open().write.assert_any_call("Foo\nBar") 1405 file_open().write.assert_any_call("Foo\nBar")
1407 m.report.to_json.assert_called_once_with() 1406 m.report.to_json.assert_called_once_with()
1408 report.merge.assert_called_with(store.Portfolio.report)
1409 1407
1410 report.reset_mock() 1408 m = market.Market(self.ccxt, self.market_args(), report_path="error", user_id=1)
1411
1412 m = market.Market(self.ccxt, report_path="error", user_id=1)
1413 with self.subTest(file="error"),\ 1409 with self.subTest(file="error"),\
1414 mock.patch("market.open") as file_open,\ 1410 mock.patch("market.open") as file_open,\
1415 mock.patch.object(m, "report") as report,\ 1411 mock.patch.object(m, "report") as report,\
1416 mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock: 1412 mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock:
1417 file_open.side_effect = FileNotFoundError 1413 file_open.side_effect = FileNotFoundError
1418 1414
1415 m.store_file_report(datetime.datetime(2018, 2, 25))
1416
1417 self.assertRegex(stdout_mock.getvalue(), "impossible to store report file: FileNotFoundError;")
1418
1419 @mock.patch.object(market, "psycopg2")
1420 def test_store_database_report(self, psycopg2):
1421 connect_mock = mock.Mock()
1422 cursor_mock = mock.MagicMock()
1423
1424 connect_mock.cursor.return_value = cursor_mock
1425 psycopg2.connect.return_value = connect_mock
1426 m = market.Market(self.ccxt, self.market_args(),
1427 pg_config={"config": "pg_config"}, user_id=1)
1428 cursor_mock.fetchone.return_value = [42]
1429
1430 with self.subTest(error=False),\
1431 mock.patch.object(m, "report") as report:
1432 report.to_json_array.return_value = [
1433 ("date1", "type1", "payload1"),
1434 ("date2", "type2", "payload2"),
1435 ]
1436 m.store_database_report(datetime.datetime(2018, 3, 24))
1437 connect_mock.assert_has_calls([
1438 mock.call.cursor(),
1439 mock.call.cursor().execute('INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id;', (datetime.datetime(2018, 3, 24), None, False)),
1440 mock.call.cursor().fetchone(),
1441 mock.call.cursor().execute('INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);', ('date1', 42, 'type1', 'payload1')),
1442 mock.call.cursor().execute('INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);', ('date2', 42, 'type2', 'payload2')),
1443 mock.call.commit(),
1444 mock.call.cursor().close(),
1445 mock.call.close()
1446 ])
1447
1448 connect_mock.reset_mock()
1449 with self.subTest(error=True),\
1450 mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock:
1451 psycopg2.connect.side_effect = Exception("Bouh")
1452 m.store_database_report(datetime.datetime(2018, 3, 24))
1453 self.assertEqual(stdout_mock.getvalue(), "impossible to store report to database: Exception; Bouh\n")
1454
1455 def test_store_report(self):
1456 m = market.Market(self.ccxt, self.market_args(), user_id=1)
1457 with self.subTest(file=None, pg_config=None),\
1458 mock.patch.object(m, "report") as report,\
1459 mock.patch.object(m, "store_database_report") as db_report,\
1460 mock.patch.object(m, "store_file_report") as file_report:
1461 m.store_report()
1462 report.merge.assert_called_with(store.Portfolio.report)
1463
1464 file_report.assert_not_called()
1465 db_report.assert_not_called()
1466
1467 report.reset_mock()
1468 m = market.Market(self.ccxt, self.market_args(), report_path="present", user_id=1)
1469 with self.subTest(file="present", pg_config=None),\
1470 mock.patch.object(m, "report") as report,\
1471 mock.patch.object(m, "store_file_report") as file_report,\
1472 mock.patch.object(m, "store_database_report") as db_report,\
1473 mock.patch.object(market, "datetime") as time_mock:
1474
1475 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
1476
1419 m.store_report() 1477 m.store_report()
1420 1478
1421 report.merge.assert_called_with(store.Portfolio.report) 1479 report.merge.assert_called_with(store.Portfolio.report)
1422 self.assertRegex(stdout_mock.getvalue(), "impossible to store report file: FileNotFoundError;") 1480 file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
1481 db_report.assert_not_called()
1482
1483 report.reset_mock()
1484 m = market.Market(self.ccxt, self.market_args(), pg_config="present", user_id=1)
1485 with self.subTest(file=None, pg_config="present"),\
1486 mock.patch.object(m, "report") as report,\
1487 mock.patch.object(m, "store_file_report") as file_report,\
1488 mock.patch.object(m, "store_database_report") as db_report,\
1489 mock.patch.object(market, "datetime") as time_mock:
1490
1491 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
1492
1493 m.store_report()
1494
1495 report.merge.assert_called_with(store.Portfolio.report)
1496 file_report.assert_not_called()
1497 db_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
1498
1499 report.reset_mock()
1500 m = market.Market(self.ccxt, self.market_args(),
1501 pg_config="pg_config", report_path="present", user_id=1)
1502 with self.subTest(file="present", pg_config="present"),\
1503 mock.patch.object(m, "report") as report,\
1504 mock.patch.object(m, "store_file_report") as file_report,\
1505 mock.patch.object(m, "store_database_report") as db_report,\
1506 mock.patch.object(market, "datetime") as time_mock:
1507
1508 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
1509
1510 m.store_report()
1511
1512 report.merge.assert_called_with(store.Portfolio.report)
1513 file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
1514 db_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
1423 1515
1424 def test_print_orders(self): 1516 def test_print_orders(self):
1425 m = market.Market(self.ccxt) 1517 m = market.Market(self.ccxt, self.market_args())
1426 with mock.patch.object(m.report, "log_stage") as log_stage,\ 1518 with mock.patch.object(m.report, "log_stage") as log_stage,\
1427 mock.patch.object(m.balances, "fetch_balances") as fetch_balances,\ 1519 mock.patch.object(m.balances, "fetch_balances") as fetch_balances,\
1428 mock.patch.object(m, "prepare_trades") as prepare_trades,\ 1520 mock.patch.object(m, "prepare_trades") as prepare_trades,\
@@ -1436,7 +1528,7 @@ class MarketTest(WebMockTestCase):
1436 prepare_orders.assert_called_with(compute_value="average") 1528 prepare_orders.assert_called_with(compute_value="average")
1437 1529
1438 def test_print_balances(self): 1530 def test_print_balances(self):
1439 m = market.Market(self.ccxt) 1531 m = market.Market(self.ccxt, self.market_args())
1440 1532
1441 with mock.patch.object(m.balances, "in_currency") as in_currency,\ 1533 with mock.patch.object(m.balances, "in_currency") as in_currency,\
1442 mock.patch.object(m.report, "log_stage") as log_stage,\ 1534 mock.patch.object(m.report, "log_stage") as log_stage,\
@@ -1461,7 +1553,7 @@ class MarketTest(WebMockTestCase):
1461 @mock.patch("market.ReportStore.log_error") 1553 @mock.patch("market.ReportStore.log_error")
1462 @mock.patch("market.Market.store_report") 1554 @mock.patch("market.Market.store_report")
1463 def test_process(self, store_report, log_error, process): 1555 def test_process(self, store_report, log_error, process):
1464 m = market.Market(self.ccxt) 1556 m = market.Market(self.ccxt, self.market_args())
1465 with self.subTest(before=False, after=False): 1557 with self.subTest(before=False, after=False):
1466 m.process(None) 1558 m.process(None)
1467 1559
@@ -3039,6 +3131,14 @@ class ReportStoreTest(WebMockTestCase):
3039 report_store.print_log(portfolio.Amount("BTC", 1)) 3131 report_store.print_log(portfolio.Amount("BTC", 1))
3040 self.assertEqual(stdout_mock.getvalue(), "") 3132 self.assertEqual(stdout_mock.getvalue(), "")
3041 3133
3134 def test_default_json_serial(self):
3135 report_store = market.ReportStore(self.m)
3136
3137 self.assertEqual("2018-02-24T00:00:00",
3138 report_store.default_json_serial(portfolio.datetime(2018, 2, 24)))
3139 self.assertEqual("1.00000000 BTC",
3140 report_store.default_json_serial(portfolio.Amount("BTC", 1)))
3141
3042 def test_to_json(self): 3142 def test_to_json(self):
3043 report_store = market.ReportStore(self.m) 3143 report_store = market.ReportStore(self.m)
3044 report_store.logs.append({"foo": "bar"}) 3144 report_store.logs.append({"foo": "bar"})
@@ -3048,6 +3148,20 @@ class ReportStoreTest(WebMockTestCase):
3048 report_store.logs.append({"amount": portfolio.Amount("BTC", 1)}) 3148 report_store.logs.append({"amount": portfolio.Amount("BTC", 1)})
3049 self.assertEqual('[\n {\n "foo": "bar"\n },\n {\n "date": "2018-02-24T00:00:00"\n },\n {\n "amount": "1.00000000 BTC"\n }\n]', report_store.to_json()) 3149 self.assertEqual('[\n {\n "foo": "bar"\n },\n {\n "date": "2018-02-24T00:00:00"\n },\n {\n "amount": "1.00000000 BTC"\n }\n]', report_store.to_json())
3050 3150
3151 def test_to_json_array(self):
3152 report_store = market.ReportStore(self.m)
3153 report_store.logs.append({
3154 "date": "date1", "type": "type1", "foo": "bar", "bla": "bla"
3155 })
3156 report_store.logs.append({
3157 "date": "date2", "type": "type2", "foo": "bar", "bla": "bla"
3158 })
3159 logs = list(report_store.to_json_array())
3160
3161 self.assertEqual(2, len(logs))
3162 self.assertEqual(("date1", "type1", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[0])
3163 self.assertEqual(("date2", "type2", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[1])
3164
3051 @mock.patch.object(market.ReportStore, "print_log") 3165 @mock.patch.object(market.ReportStore, "print_log")
3052 @mock.patch.object(market.ReportStore, "add_log") 3166 @mock.patch.object(market.ReportStore, "add_log")
3053 def test_log_stage(self, add_log, print_log): 3167 def test_log_stage(self, add_log, print_log):
@@ -3541,7 +3655,7 @@ class MainTest(WebMockTestCase):
3541 mock.patch("main.parse_config") as main_parse_config: 3655 mock.patch("main.parse_config") as main_parse_config:
3542 with self.subTest(debug=False): 3656 with self.subTest(debug=False):
3543 main_parse_config.return_value = ["pg_config", "report_path"] 3657 main_parse_config.return_value = ["pg_config", "report_path"]
3544 main_fetch_markets.return_value = [({"key": "market_config"},)] 3658 main_fetch_markets.return_value = [(1, {"key": "market_config"}, 3)]
3545 m = main.get_user_market("config_path.ini", 1) 3659 m = main.get_user_market("config_path.ini", 1)
3546 3660
3547 self.assertIsInstance(m, market.Market) 3661 self.assertIsInstance(m, market.Market)
@@ -3549,7 +3663,7 @@ class MainTest(WebMockTestCase):
3549 3663
3550 with self.subTest(debug=True): 3664 with self.subTest(debug=True):
3551 main_parse_config.return_value = ["pg_config", "report_path"] 3665 main_parse_config.return_value = ["pg_config", "report_path"]
3552 main_fetch_markets.return_value = [({"key": "market_config"},)] 3666 main_fetch_markets.return_value = [(1, {"key": "market_config"}, 3)]
3553 m = main.get_user_market("config_path.ini", 1, debug=True) 3667 m = main.get_user_market("config_path.ini", 1, debug=True)
3554 3668
3555 self.assertIsInstance(m, market.Market) 3669 self.assertIsInstance(m, market.Market)
@@ -3568,16 +3682,16 @@ class MainTest(WebMockTestCase):
3568 args_mock.after = "after" 3682 args_mock.after = "after"
3569 self.assertEqual("", stdout_mock.getvalue()) 3683 self.assertEqual("", stdout_mock.getvalue())
3570 3684
3571 main.process("config", 1, "report_path", args_mock) 3685 main.process("config", 3, 1, args_mock, "report_path", "pg_config")
3572 3686
3573 market_mock.from_config.assert_has_calls([ 3687 market_mock.from_config.assert_has_calls([
3574 mock.call("config", debug="debug", user_id=1, report_path="report_path"), 3688 mock.call("config", args_mock, pg_config="pg_config", market_id=3, user_id=1, report_path="report_path"),
3575 mock.call().process("action", before="before", after="after"), 3689 mock.call().process("action", before="before", after="after"),
3576 ]) 3690 ])
3577 3691
3578 with self.subTest(exception=True): 3692 with self.subTest(exception=True):
3579 market_mock.from_config.side_effect = Exception("boo") 3693 market_mock.from_config.side_effect = Exception("boo")
3580 main.process("config", 1, "report_path", args_mock) 3694 main.process(3, "config", 1, "report_path", args_mock, "pg_config")
3581 self.assertEqual("Exception: boo\n", stdout_mock.getvalue()) 3695 self.assertEqual("Exception: boo\n", stdout_mock.getvalue())
3582 3696
3583 def test_main(self): 3697 def test_main(self):
@@ -3595,7 +3709,7 @@ class MainTest(WebMockTestCase):
3595 3709
3596 parse_config.return_value = ["pg_config", "report_path"] 3710 parse_config.return_value = ["pg_config", "report_path"]
3597 3711
3598 fetch_markets.return_value = [["config1", 1], ["config2", 2]] 3712 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
3599 3713
3600 main.main(["Foo", "Bar"]) 3714 main.main(["Foo", "Bar"])
3601 3715
@@ -3605,8 +3719,8 @@ class MainTest(WebMockTestCase):
3605 3719
3606 self.assertEqual(2, process.call_count) 3720 self.assertEqual(2, process.call_count)
3607 process.assert_has_calls([ 3721 process.assert_has_calls([
3608 mock.call("config1", 1, "report_path", args_mock), 3722 mock.call("config1", 3, 1, args_mock, "report_path", "pg_config"),
3609 mock.call("config2", 2, "report_path", args_mock), 3723 mock.call("config2", 1, 2, args_mock, "report_path", "pg_config"),
3610 ]) 3724 ])
3611 with self.subTest(parallel=True): 3725 with self.subTest(parallel=True):
3612 with mock.patch("main.parse_args") as parse_args,\ 3726 with mock.patch("main.parse_args") as parse_args,\
@@ -3623,7 +3737,7 @@ class MainTest(WebMockTestCase):
3623 3737
3624 parse_config.return_value = ["pg_config", "report_path"] 3738 parse_config.return_value = ["pg_config", "report_path"]
3625 3739
3626 fetch_markets.return_value = [["config1", 1], ["config2", 2]] 3740 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
3627 3741
3628 main.main(["Foo", "Bar"]) 3742 main.main(["Foo", "Bar"])
3629 3743
@@ -3635,9 +3749,9 @@ class MainTest(WebMockTestCase):
3635 self.assertEqual(2, process.call_count) 3749 self.assertEqual(2, process.call_count)
3636 process.assert_has_calls([ 3750 process.assert_has_calls([
3637 mock.call.__bool__(), 3751 mock.call.__bool__(),
3638 mock.call("config1", 1, "report_path", args_mock), 3752 mock.call("config1", 3, 1, args_mock, "report_path", "pg_config"),
3639 mock.call.__bool__(), 3753 mock.call.__bool__(),
3640 mock.call("config2", 2, "report_path", args_mock), 3754 mock.call("config2", 1, 2, args_mock, "report_path", "pg_config"),
3641 ]) 3755 ])
3642 3756
3643 @mock.patch.object(main.sys, "exit") 3757 @mock.patch.object(main.sys, "exit")
@@ -3723,7 +3837,7 @@ class MainTest(WebMockTestCase):
3723 rows = list(main.fetch_markets({"foo": "bar"}, None)) 3837 rows = list(main.fetch_markets({"foo": "bar"}, None))
3724 3838
3725 psycopg2.connect.assert_called_once_with(foo="bar") 3839 psycopg2.connect.assert_called_once_with(foo="bar")
3726 cursor_mock.execute.assert_called_once_with("SELECT config,user_id FROM market_configs") 3840 cursor_mock.execute.assert_called_once_with("SELECT id,config,user_id FROM market_configs")
3727 3841
3728 self.assertEqual(["row_1", "row_2"], rows) 3842 self.assertEqual(["row_1", "row_2"], rows)
3729 3843
@@ -3733,7 +3847,7 @@ class MainTest(WebMockTestCase):
3733 rows = list(main.fetch_markets({"foo": "bar"}, 1)) 3847 rows = list(main.fetch_markets({"foo": "bar"}, 1))
3734 3848
3735 psycopg2.connect.assert_called_once_with(foo="bar") 3849 psycopg2.connect.assert_called_once_with(foo="bar")
3736 cursor_mock.execute.assert_called_once_with("SELECT config,user_id FROM market_configs WHERE user_id = %s", 1) 3850 cursor_mock.execute.assert_called_once_with("SELECT id,config,user_id FROM market_configs WHERE user_id = %s", 1)
3737 3851
3738 self.assertEqual(["row_1", "row_2"], rows) 3852 self.assertEqual(["row_1", "row_2"], rows)
3739 3853
@@ -3797,7 +3911,7 @@ class ProcessorTest(WebMockTestCase):
3797 3911
3798 def test_method_arguments(self): 3912 def test_method_arguments(self):
3799 ccxt = mock.Mock(spec=market.ccxt.poloniexE) 3913 ccxt = mock.Mock(spec=market.ccxt.poloniexE)
3800 m = market.Market(ccxt) 3914 m = market.Market(ccxt, self.market_args())
3801 3915
3802 processor = market.Processor(m) 3916 processor = market.Processor(m)
3803 3917