aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--main.py42
-rw-r--r--market.py68
-rw-r--r--requirements.txt1
-rw-r--r--store.py46
-rw-r--r--tests/test_main.py53
-rw-r--r--tests/test_market.py192
-rw-r--r--tests/test_store.py47
7 files changed, 321 insertions, 128 deletions
diff --git a/main.py b/main.py
index 2c7b570..13c2240 100644
--- a/main.py
+++ b/main.py
@@ -95,13 +95,25 @@ def parse_config(args):
95 del(args.db_password) 95 del(args.db_password)
96 del(args.db_database) 96 del(args.db_database)
97 97
98 redis_config = {
99 "host": args.redis_host,
100 "port": args.redis_port,
101 "db": args.redis_database,
102 }
103 if redis_config["host"].startswith("/"):
104 redis_config["unix_socket_path"] = redis_config.pop("host")
105 del(redis_config["port"])
106 del(args.redis_host)
107 del(args.redis_port)
108 del(args.redis_database)
109
98 report_path = args.report_path 110 report_path = args.report_path
99 111
100 if report_path is not None and not \ 112 if report_path is not None and not \
101 os.path.exists(report_path): 113 os.path.exists(report_path):
102 os.makedirs(report_path) 114 os.makedirs(report_path)
103 115
104 return pg_config 116 return pg_config, redis_config
105 117
106def parse_args(argv): 118def parse_args(argv):
107 parser = configargparse.ArgumentParser( 119 parser = configargparse.ArgumentParser(
@@ -134,6 +146,10 @@ def parse_args(argv):
134 help="Store report to database (default)") 146 help="Store report to database (default)")
135 parser.add_argument("--no-report-db", action='store_false', dest="report_db", 147 parser.add_argument("--no-report-db", action='store_false', dest="report_db",
136 help="Don't store report to database") 148 help="Don't store report to database")
149 parser.add_argument("--report-redis", action='store_true', default=False, dest="report_redis",
150 help="Store report to redis")
151 parser.add_argument("--no-report-redis", action='store_false', dest="report_redis",
152 help="Don't store report to redis (default)")
137 parser.add_argument("--report-path", required=False, 153 parser.add_argument("--report-path", required=False,
138 help="Where to store the reports (default: absent, don't store)") 154 help="Where to store the reports (default: absent, don't store)")
139 parser.add_argument("--no-report-path", action='store_const', dest='report_path', const=None, 155 parser.add_argument("--no-report-path", action='store_const', dest='report_path', const=None,
@@ -148,14 +164,24 @@ def parse_args(argv):
148 help="Password access to database (default: cryptoportfolio)") 164 help="Password access to database (default: cryptoportfolio)")
149 parser.add_argument("--db-database", default="cryptoportfolio", 165 parser.add_argument("--db-database", default="cryptoportfolio",
150 help="Database access to database (default: cryptoportfolio)") 166 help="Database access to database (default: cryptoportfolio)")
151 167 parser.add_argument("--redis-host", default="localhost",
152 return parser.parse_args(argv) 168 help="Host access to database (default: localhost). Use path for socket")
153 169 parser.add_argument("--redis-port", default=6379,
154def process(market_config, market_id, user_id, args, pg_config): 170 help="Port access to redis (default: 6379)")
171 parser.add_argument("--redis-database", default=0,
172 help="Redis database to use (default: 0)")
173
174 parsed = parser.parse_args(argv)
175 if parsed.action is None:
176 parsed.action = ["sell_all"]
177 return parsed
178
179def process(market_config, market_id, user_id, args, pg_config, redis_config):
155 try: 180 try:
156 market.Market\ 181 market.Market\
157 .from_config(market_config, args, market_id=market_id, 182 .from_config(market_config, args, market_id=market_id,
158 pg_config=pg_config, user_id=user_id)\ 183 pg_config=pg_config, redis_config=redis_config,
184 user_id=user_id)\
159 .process(args.action, before=args.before, after=args.after) 185 .process(args.action, before=args.before, after=args.after)
160 except Exception as e: 186 except Exception as e:
161 print("{}: {}".format(e.__class__.__name__, e)) 187 print("{}: {}".format(e.__class__.__name__, e))
@@ -163,7 +189,7 @@ def process(market_config, market_id, user_id, args, pg_config):
163def main(argv): 189def main(argv):
164 args = parse_args(argv) 190 args = parse_args(argv)
165 191
166 pg_config = parse_config(args) 192 pg_config, redis_config = parse_config(args)
167 193
168 market.Portfolio.report.set_verbose(not args.quiet) 194 market.Portfolio.report.set_verbose(not args.quiet)
169 195
@@ -180,7 +206,7 @@ def main(argv):
180 process_ = process 206 process_ = process
181 207
182 for market_id, market_config, user_id in fetch_markets(pg_config, args.user): 208 for market_id, market_config, user_id in fetch_markets(pg_config, args.user):
183 process_(market_config, market_id, user_id, args, pg_config) 209 process_(market_config, market_id, user_id, args, pg_config, redis_config)
184 210
185 if args.parallel: 211 if args.parallel:
186 for thread in threads: 212 for thread in threads:
diff --git a/market.py b/market.py
index 7a37cf6..ce0c48c 100644
--- a/market.py
+++ b/market.py
@@ -2,6 +2,7 @@ from ccxt import ExchangeError, NotSupported, RequestTimeout, InvalidNonce
2import ccxt_wrapper as ccxt 2import ccxt_wrapper as ccxt
3import time 3import time
4import psycopg2 4import psycopg2
5import redis
5from store import * 6from store import *
6from cachetools.func import ttl_cache 7from cachetools.func import ttl_cache
7from datetime import datetime 8from datetime import datetime
@@ -26,7 +27,7 @@ class Market:
26 self.balances = BalanceStore(self) 27 self.balances = BalanceStore(self)
27 self.processor = Processor(self) 28 self.processor = Processor(self)
28 29
29 for key in ["user_id", "market_id", "pg_config"]: 30 for key in ["user_id", "market_id", "pg_config", "redis_config"]:
30 setattr(self, key, kwargs.get(key, None)) 31 setattr(self, key, kwargs.get(key, None))
31 32
32 self.report.log_market(self.args) 33 self.report.log_market(self.args)
@@ -46,6 +47,8 @@ class Market:
46 self.store_file_report(date) 47 self.store_file_report(date)
47 if self.pg_config is not None and self.args.report_db: 48 if self.pg_config is not None and self.args.report_db:
48 self.store_database_report(date) 49 self.store_database_report(date)
50 if self.redis_config is not None and self.args.report_redis:
51 self.store_redis_report(date)
49 52
50 def store_file_report(self, date): 53 def store_file_report(self, date):
51 try: 54 try:
@@ -74,19 +77,26 @@ class Market:
74 except Exception as e: 77 except Exception as e:
75 print("impossible to store report to database: {}; {}".format(e.__class__.__name__, e)) 78 print("impossible to store report to database: {}; {}".format(e.__class__.__name__, e))
76 79
80 def store_redis_report(self, date):
81 try:
82 conn = redis.Redis(**self.redis_config)
83 for type_, log in self.report.to_json_redis():
84 key = "/cryptoportfolio/{}/{}/{}".format(self.market_id, date.isoformat(), type_)
85 conn.set(key, log, ex=31*24*60*60)
86 key = "/cryptoportfolio/{}/latest/{}".format(self.market_id, type_)
87 conn.set(key, log)
88 except Exception as e:
89 print("impossible to store report to redis: {}; {}".format(e.__class__.__name__, e))
90
77 def process(self, actions, before=False, after=False): 91 def process(self, actions, before=False, after=False):
78 try: 92 try:
79 if len(actions or []) == 0: 93 for action in actions:
80 if before: 94 if bool(before) is bool(after):
81 self.processor.process("sell_all", steps="before") 95 self.processor.process(action, steps="all")
82 if after: 96 elif before:
83 self.processor.process("sell_all", steps="after") 97 self.processor.process(action, steps="before")
84 else: 98 elif after:
85 for action in actions: 99 self.processor.process(action, steps="after")
86 if hasattr(self, action):
87 getattr(self, action)()
88 else:
89 self.report.log_error("market_process", message="Unknown action {}".format(action))
90 except Exception as e: 100 except Exception as e:
91 self.report.log_error("market_process", exception=e) 101 self.report.log_error("market_process", exception=e)
92 finally: 102 finally:
@@ -212,16 +222,7 @@ class Market:
212 liquidity=liquidity, repartition=repartition) 222 liquidity=liquidity, repartition=repartition)
213 self.trades.compute_trades(values_in_base, new_repartition, only=only) 223 self.trades.compute_trades(values_in_base, new_repartition, only=only)
214 224
215 # Helpers 225 def print_tickers(self, base_currency="BTC"):
216 def print_orders(self, base_currency="BTC"):
217 self.report.log_stage("print_orders")
218 self.balances.fetch_balances(tag="print_orders")
219 self.prepare_trades(base_currency=base_currency, compute_value="average")
220 self.trades.prepare_orders(compute_value="average")
221
222 def print_balances(self, base_currency="BTC"):
223 self.report.log_stage("print_balances")
224 self.balances.fetch_balances()
225 if base_currency is not None: 226 if base_currency is not None:
226 self.report.print_log("total:") 227 self.report.print_log("total:")
227 self.report.print_log(sum(self.balances.in_currency(base_currency).values())) 228 self.report.print_log(sum(self.balances.in_currency(base_currency).values()))
@@ -237,12 +238,20 @@ class Processor:
237 "wait_for_recent": {}, 238 "wait_for_recent": {},
238 }, 239 },
239 ], 240 ],
241 "print_balances": [
242 {
243 "name": "print_balances",
244 "number": 1,
245 "fetch_balances": ["begin"],
246 "print_tickers": { "base_currency": "BTC" },
247 }
248 ],
240 "print_orders": [ 249 "print_orders": [
241 { 250 {
242 "name": "wait", 251 "name": "wait",
243 "number": 1, 252 "number": 1,
244 "before": False, 253 "before": True,
245 "after": True, 254 "after": False,
246 "wait_for_recent": {}, 255 "wait_for_recent": {},
247 }, 256 },
248 { 257 {
@@ -328,7 +337,7 @@ class Processor:
328 ordered_actions = [ 337 ordered_actions = [
329 "wait_for_recent", "prepare_trades", "prepare_orders", 338 "wait_for_recent", "prepare_trades", "prepare_orders",
330 "move_balances", "run_orders", "follow_orders", 339 "move_balances", "run_orders", "follow_orders",
331 "close_trades"] 340 "close_trades", "print_tickers"]
332 341
333 def __init__(self, market): 342 def __init__(self, market):
334 self.market = market 343 self.market = market
@@ -337,7 +346,7 @@ class Processor:
337 if step == "all": 346 if step == "all":
338 return scenario 347 return scenario
339 elif step == "before" or step == "after": 348 elif step == "before" or step == "after":
340 return list(filter(lambda x: step in x and x[step], scenario)) 349 return list(filter(lambda x: x.get(step, False), scenario))
341 elif type(step) == int: 350 elif type(step) == int:
342 return [scenario[step-1]] 351 return [scenario[step-1]]
343 elif type(step) == str: 352 elif type(step) == str:
@@ -345,7 +354,12 @@ class Processor:
345 else: 354 else:
346 raise TypeError("Unknown step {}".format(step)) 355 raise TypeError("Unknown step {}".format(step))
347 356
357 def can_process(self, scenario_name):
358 return scenario_name in self.scenarios
359
348 def process(self, scenario_name, steps="all", **kwargs): 360 def process(self, scenario_name, steps="all", **kwargs):
361 if not self.can_process(scenario_name):
362 raise TypeError("Unknown scenario {}".format(scenario_name))
349 scenario = self.scenarios[scenario_name] 363 scenario = self.scenarios[scenario_name]
350 selected_steps = [] 364 selected_steps = []
351 365
@@ -388,6 +402,8 @@ class Processor:
388 method = self.market.follow_orders 402 method = self.market.follow_orders
389 elif action == "close_trades": 403 elif action == "close_trades":
390 method = self.market.trades.close_trades 404 method = self.market.trades.close_trades
405 elif action == "print_tickers":
406 method = self.market.print_tickers
391 407
392 signature = inspect.getfullargspec(method) 408 signature = inspect.getfullargspec(method)
393 defaults = signature.defaults or [] 409 defaults = signature.defaults or []
diff --git a/requirements.txt b/requirements.txt
index 2451c80..3a4db2d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ psycopg2==2.7.4
6retry==0.9.2 6retry==0.9.2
7cachetools==2.0.1 7cachetools==2.0.1
8configargparse==0.12.0 8configargparse==0.12.0
9redis==2.10.6
diff --git a/store.py b/store.py
index 0c018e0..072d3a2 100644
--- a/store.py
+++ b/store.py
@@ -17,6 +17,7 @@ class ReportStore:
17 17
18 self.print_logs = [] 18 self.print_logs = []
19 self.logs = [] 19 self.logs = []
20 self.redis_status = []
20 21
21 self.no_http_dup = no_http_dup 22 self.no_http_dup = no_http_dup
22 self.last_http = None 23 self.last_http = None
@@ -46,6 +47,10 @@ class ReportStore:
46 self.logs.append(hash_) 47 self.logs.append(hash_)
47 return hash_ 48 return hash_
48 49
50 def add_redis_status(self, hash_):
51 self.redis_status.append(hash_)
52 return hash_
53
49 @staticmethod 54 @staticmethod
50 def default_json_serial(obj): 55 def default_json_serial(obj):
51 if isinstance(obj, (datetime.datetime, datetime.date)): 56 if isinstance(obj, (datetime.datetime, datetime.date)):
@@ -63,6 +68,13 @@ class ReportStore:
63 json.dumps(log, default=self.default_json_serial, indent=" ") 68 json.dumps(log, default=self.default_json_serial, indent=" ")
64 ) 69 )
65 70
71 def to_json_redis(self):
72 for log in (x.copy() for x in self.redis_status):
73 yield (
74 log.pop("type"),
75 json.dumps(log, default=self.default_json_serial)
76 )
77
66 def set_verbose(self, verbose_print): 78 def set_verbose(self, verbose_print):
67 self.verbose_print = verbose_print 79 self.verbose_print = verbose_print
68 80
@@ -91,11 +103,14 @@ class ReportStore:
91 for currency, balance in self.market.balances.all.items(): 103 for currency, balance in self.market.balances.all.items():
92 self.print_log("\t{}".format(balance)) 104 self.print_log("\t{}".format(balance))
93 105
94 self.add_log({ 106 log = {
95 "type": "balance", 107 "type": "balance",
96 "tag": tag, 108 "tag": tag,
97 "balances": self.market.balances.as_json() 109 "balances": self.market.balances.as_json()
98 }) 110 }
111
112 self.add_log(log.copy())
113 self.add_redis_status(log)
99 114
100 def log_tickers(self, amounts, other_currency, 115 def log_tickers(self, amounts, other_currency,
101 compute_value, type): 116 compute_value, type):
@@ -107,15 +122,18 @@ class ReportStore:
107 for currency, amount in amounts.items(): 122 for currency, amount in amounts.items():
108 values[currency] = amount.as_json()["value"] 123 values[currency] = amount.as_json()["value"]
109 rates[currency] = amount.rate 124 rates[currency] = amount.rate
110 self.add_log({ 125 log = {
111 "type": "tickers", 126 "type": "tickers",
112 "compute_value": compute_value, 127 "compute_value": compute_value,
113 "balance_type": type, 128 "balance_type": type,
114 "currency": other_currency, 129 "currency": other_currency,
115 "balances": values, 130 "balances": values,
116 "rates": rates, 131 "rates": rates,
117 "total": sum(amounts.values()).as_json()["value"] 132 "total": sum(amounts.values()).as_json()["value"]
118 }) 133 }
134
135 self.add_log(log.copy())
136 self.add_redis_status(log)
119 137
120 def log_dispatch(self, amount, amounts, liquidity, repartition): 138 def log_dispatch(self, amount, amounts, liquidity, repartition):
121 self.add_log({ 139 self.add_log({
diff --git a/tests/test_main.py b/tests/test_main.py
index d2f8029..b650870 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -135,16 +135,16 @@ class MainTest(WebMockTestCase):
135 args_mock.after = "after" 135 args_mock.after = "after"
136 self.assertEqual("", stdout_mock.getvalue()) 136 self.assertEqual("", stdout_mock.getvalue())
137 137
138 main.process("config", 3, 1, args_mock, "pg_config") 138 main.process("config", 3, 1, args_mock, "pg_config", "redis_config")
139 139
140 market_mock.from_config.assert_has_calls([ 140 market_mock.from_config.assert_has_calls([
141 mock.call("config", args_mock, pg_config="pg_config", market_id=3, user_id=1), 141 mock.call("config", args_mock, pg_config="pg_config", redis_config="redis_config", market_id=3, user_id=1),
142 mock.call().process("action", before="before", after="after"), 142 mock.call().process("action", before="before", after="after"),
143 ]) 143 ])
144 144
145 with self.subTest(exception=True): 145 with self.subTest(exception=True):
146 market_mock.from_config.side_effect = Exception("boo") 146 market_mock.from_config.side_effect = Exception("boo")
147 main.process(3, "config", 1, args_mock, "pg_config") 147 main.process(3, "config", 1, args_mock, "pg_config", "redis_config")
148 self.assertEqual("Exception: boo\n", stdout_mock.getvalue()) 148 self.assertEqual("Exception: boo\n", stdout_mock.getvalue())
149 149
150 def test_main(self): 150 def test_main(self):
@@ -159,7 +159,7 @@ class MainTest(WebMockTestCase):
159 args_mock.user = "user" 159 args_mock.user = "user"
160 parse_args.return_value = args_mock 160 parse_args.return_value = args_mock
161 161
162 parse_config.return_value = "pg_config" 162 parse_config.return_value = ["pg_config", "redis_config"]
163 163
164 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] 164 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
165 165
@@ -171,8 +171,8 @@ class MainTest(WebMockTestCase):
171 171
172 self.assertEqual(2, process.call_count) 172 self.assertEqual(2, process.call_count)
173 process.assert_has_calls([ 173 process.assert_has_calls([
174 mock.call("config1", 3, 1, args_mock, "pg_config"), 174 mock.call("config1", 3, 1, args_mock, "pg_config", "redis_config"),
175 mock.call("config2", 1, 2, args_mock, "pg_config"), 175 mock.call("config2", 1, 2, args_mock, "pg_config", "redis_config"),
176 ]) 176 ])
177 with self.subTest(parallel=True): 177 with self.subTest(parallel=True):
178 with mock.patch("main.parse_args") as parse_args,\ 178 with mock.patch("main.parse_args") as parse_args,\
@@ -187,7 +187,7 @@ class MainTest(WebMockTestCase):
187 args_mock.user = "user" 187 args_mock.user = "user"
188 parse_args.return_value = args_mock 188 parse_args.return_value = args_mock
189 189
190 parse_config.return_value = "pg_config" 190 parse_config.return_value = ["pg_config", "redis_config"]
191 191
192 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] 192 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
193 193
@@ -202,9 +202,9 @@ class MainTest(WebMockTestCase):
202 self.assertEqual(2, process.call_count) 202 self.assertEqual(2, process.call_count)
203 process.assert_has_calls([ 203 process.assert_has_calls([
204 mock.call.__bool__(), 204 mock.call.__bool__(),
205 mock.call("config1", 3, 1, args_mock, "pg_config"), 205 mock.call("config1", 3, 1, args_mock, "pg_config", "redis_config"),
206 mock.call.__bool__(), 206 mock.call.__bool__(),
207 mock.call("config2", 1, 2, args_mock, "pg_config"), 207 mock.call("config2", 1, 2, args_mock, "pg_config", "redis_config"),
208 ]) 208 ])
209 with self.subTest(quiet=True): 209 with self.subTest(quiet=True):
210 with mock.patch("main.parse_args") as parse_args,\ 210 with mock.patch("main.parse_args") as parse_args,\
@@ -219,7 +219,7 @@ class MainTest(WebMockTestCase):
219 args_mock.user = "user" 219 args_mock.user = "user"
220 parse_args.return_value = args_mock 220 parse_args.return_value = args_mock
221 221
222 parse_config.return_value = "pg_config" 222 parse_config.return_value = ["pg_config", "redis_config"]
223 223
224 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] 224 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
225 225
@@ -240,7 +240,7 @@ class MainTest(WebMockTestCase):
240 args_mock.user = "user" 240 args_mock.user = "user"
241 parse_args.return_value = args_mock 241 parse_args.return_value = args_mock
242 242
243 parse_config.return_value = "pg_config" 243 parse_config.return_value = ["pg_config", "redis_config"]
244 244
245 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] 245 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
246 246
@@ -259,15 +259,39 @@ class MainTest(WebMockTestCase):
259 "db_user": "user", 259 "db_user": "user",
260 "db_password": "password", 260 "db_password": "password",
261 "db_database": "database", 261 "db_database": "database",
262 "redis_host": "rhost",
263 "redis_port": "rport",
264 "redis_database": "rdb",
262 "report_path": None, 265 "report_path": None,
263 }) 266 })
264 267
265 result = main.parse_config(args) 268 db_config, redis_config = main.parse_config(args)
266 self.assertEqual({ "host": "host", "port": "port", "user": 269 self.assertEqual({ "host": "host", "port": "port", "user":
267 "user", "password": "password", "database": "database" 270 "user", "password": "password", "database": "database"
268 }, result) 271 }, db_config)
272 self.assertEqual({ "host": "rhost", "port": "rport", "db":
273 "rdb"}, redis_config)
274
269 with self.assertRaises(AttributeError): 275 with self.assertRaises(AttributeError):
270 args.db_password 276 args.db_password
277 with self.assertRaises(AttributeError):
278 args.redis_host
279
280 with self.subTest(redis_host="socket"):
281 args = main.configargparse.Namespace(**{
282 "db_host": "host",
283 "db_port": "port",
284 "db_user": "user",
285 "db_password": "password",
286 "db_database": "database",
287 "redis_host": "/run/foo",
288 "redis_port": "rport",
289 "redis_database": "rdb",
290 "report_path": None,
291 })
292
293 db_config, redis_config = main.parse_config(args)
294 self.assertEqual({ "unix_socket_path": "/run/foo", "db": "rdb"}, redis_config)
271 295
272 with self.subTest(report_path="present"): 296 with self.subTest(report_path="present"):
273 args = main.configargparse.Namespace(**{ 297 args = main.configargparse.Namespace(**{
@@ -276,6 +300,9 @@ class MainTest(WebMockTestCase):
276 "db_user": "user", 300 "db_user": "user",
277 "db_password": "password", 301 "db_password": "password",
278 "db_database": "database", 302 "db_database": "database",
303 "redis_host": "rhost",
304 "redis_port": "rport",
305 "redis_database": "rdb",
279 "report_path": "report_path", 306 "report_path": "report_path",
280 }) 307 })
281 308
diff --git a/tests/test_market.py b/tests/test_market.py
index 14b23b5..e3482b8 100644
--- a/tests/test_market.py
+++ b/tests/test_market.py
@@ -530,23 +530,55 @@ class MarketTest(WebMockTestCase):
530 m.store_database_report(datetime.datetime(2018, 3, 24)) 530 m.store_database_report(datetime.datetime(2018, 3, 24))
531 self.assertEqual(stdout_mock.getvalue(), "impossible to store report to database: Exception; Bouh\n") 531 self.assertEqual(stdout_mock.getvalue(), "impossible to store report to database: Exception; Bouh\n")
532 532
533 @mock.patch.object(market, "redis")
534 def test_store_redis_report(self, redis):
535 connect_mock = mock.Mock()
536 redis.Redis.return_value = connect_mock
537
538 m = market.Market(self.ccxt, self.market_args(),
539 redis_config={"config": "redis_config"}, market_id=1)
540
541 with self.subTest(error=False),\
542 mock.patch.object(m, "report") as report:
543 report.to_json_redis.return_value = [
544 ("type1", "payload1"),
545 ("type2", "payload2"),
546 ]
547 m.store_redis_report(datetime.datetime(2018, 3, 24))
548 connect_mock.assert_has_calls([
549 mock.call.set("/cryptoportfolio/1/2018-03-24T00:00:00/type1", "payload1", ex=31*24*60*60),
550 mock.call.set("/cryptoportfolio/1/latest/type1", "payload1"),
551 mock.call.set("/cryptoportfolio/1/2018-03-24T00:00:00/type2", "payload2", ex=31*24*60*60),
552 mock.call.set("/cryptoportfolio/1/latest/type2", "payload2"),
553 ])
554
555 connect_mock.reset_mock()
556 with self.subTest(error=True),\
557 mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock:
558 redis.Redis.side_effect = Exception("Bouh")
559 m.store_redis_report(datetime.datetime(2018, 3, 24))
560 self.assertEqual(stdout_mock.getvalue(), "impossible to store report to redis: Exception; Bouh\n")
561
533 def test_store_report(self): 562 def test_store_report(self):
534 m = market.Market(self.ccxt, self.market_args(report_db=False), user_id=1) 563 m = market.Market(self.ccxt, self.market_args(report_db=False), user_id=1)
535 with self.subTest(file=None, pg_config=None),\ 564 with self.subTest(file=None, pg_config=None),\
536 mock.patch.object(m, "report") as report,\ 565 mock.patch.object(m, "report") as report,\
537 mock.patch.object(m, "store_database_report") as db_report,\ 566 mock.patch.object(m, "store_database_report") as db_report,\
567 mock.patch.object(m, "store_redis_report") as redis_report,\
538 mock.patch.object(m, "store_file_report") as file_report: 568 mock.patch.object(m, "store_file_report") as file_report:
539 m.store_report() 569 m.store_report()
540 report.merge.assert_called_with(store.Portfolio.report) 570 report.merge.assert_called_with(store.Portfolio.report)
541 571
542 file_report.assert_not_called() 572 file_report.assert_not_called()
543 db_report.assert_not_called() 573 db_report.assert_not_called()
574 redis_report.assert_not_called()
544 575
545 report.reset_mock() 576 report.reset_mock()
546 m = market.Market(self.ccxt, self.market_args(report_db=False, report_path="present"), user_id=1) 577 m = market.Market(self.ccxt, self.market_args(report_db=False, report_path="present"), user_id=1)
547 with self.subTest(file="present", pg_config=None),\ 578 with self.subTest(file="present", pg_config=None),\
548 mock.patch.object(m, "report") as report,\ 579 mock.patch.object(m, "report") as report,\
549 mock.patch.object(m, "store_file_report") as file_report,\ 580 mock.patch.object(m, "store_file_report") as file_report,\
581 mock.patch.object(m, "store_redis_report") as redis_report,\
550 mock.patch.object(m, "store_database_report") as db_report,\ 582 mock.patch.object(m, "store_database_report") as db_report,\
551 mock.patch.object(market.datetime, "datetime") as time_mock: 583 mock.patch.object(market.datetime, "datetime") as time_mock:
552 584
@@ -557,12 +589,14 @@ class MarketTest(WebMockTestCase):
557 report.merge.assert_called_with(store.Portfolio.report) 589 report.merge.assert_called_with(store.Portfolio.report)
558 file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) 590 file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
559 db_report.assert_not_called() 591 db_report.assert_not_called()
592 redis_report.assert_not_called()
560 593
561 report.reset_mock() 594 report.reset_mock()
562 m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), user_id=1) 595 m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), user_id=1)
563 with self.subTest(file="present", pg_config=None, report_db=True),\ 596 with self.subTest(file="present", pg_config=None, report_db=True),\
564 mock.patch.object(m, "report") as report,\ 597 mock.patch.object(m, "report") as report,\
565 mock.patch.object(m, "store_file_report") as file_report,\ 598 mock.patch.object(m, "store_file_report") as file_report,\
599 mock.patch.object(m, "store_redis_report") as redis_report,\
566 mock.patch.object(m, "store_database_report") as db_report,\ 600 mock.patch.object(m, "store_database_report") as db_report,\
567 mock.patch.object(market.datetime, "datetime") as time_mock: 601 mock.patch.object(market.datetime, "datetime") as time_mock:
568 602
@@ -573,12 +607,14 @@ class MarketTest(WebMockTestCase):
573 report.merge.assert_called_with(store.Portfolio.report) 607 report.merge.assert_called_with(store.Portfolio.report)
574 file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) 608 file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
575 db_report.assert_not_called() 609 db_report.assert_not_called()
610 redis_report.assert_not_called()
576 611
577 report.reset_mock() 612 report.reset_mock()
578 m = market.Market(self.ccxt, self.market_args(report_db=True), pg_config="present", user_id=1) 613 m = market.Market(self.ccxt, self.market_args(report_db=True), pg_config="present", user_id=1)
579 with self.subTest(file=None, pg_config="present"),\ 614 with self.subTest(file=None, pg_config="present"),\
580 mock.patch.object(m, "report") as report,\ 615 mock.patch.object(m, "report") as report,\
581 mock.patch.object(m, "store_file_report") as file_report,\ 616 mock.patch.object(m, "store_file_report") as file_report,\
617 mock.patch.object(m, "store_redis_report") as redis_report,\
582 mock.patch.object(m, "store_database_report") as db_report,\ 618 mock.patch.object(m, "store_database_report") as db_report,\
583 mock.patch.object(market.datetime, "datetime") as time_mock: 619 mock.patch.object(market.datetime, "datetime") as time_mock:
584 620
@@ -589,6 +625,7 @@ class MarketTest(WebMockTestCase):
589 report.merge.assert_called_with(store.Portfolio.report) 625 report.merge.assert_called_with(store.Portfolio.report)
590 file_report.assert_not_called() 626 file_report.assert_not_called()
591 db_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) 627 db_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
628 redis_report.assert_not_called()
592 629
593 report.reset_mock() 630 report.reset_mock()
594 m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), 631 m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"),
@@ -596,6 +633,7 @@ class MarketTest(WebMockTestCase):
596 with self.subTest(file="present", pg_config="present"),\ 633 with self.subTest(file="present", pg_config="present"),\
597 mock.patch.object(m, "report") as report,\ 634 mock.patch.object(m, "report") as report,\
598 mock.patch.object(m, "store_file_report") as file_report,\ 635 mock.patch.object(m, "store_file_report") as file_report,\
636 mock.patch.object(m, "store_redis_report") as redis_report,\
599 mock.patch.object(m, "store_database_report") as db_report,\ 637 mock.patch.object(m, "store_database_report") as db_report,\
600 mock.patch.object(market.datetime, "datetime") as time_mock: 638 mock.patch.object(market.datetime, "datetime") as time_mock:
601 639
@@ -606,22 +644,54 @@ class MarketTest(WebMockTestCase):
606 report.merge.assert_called_with(store.Portfolio.report) 644 report.merge.assert_called_with(store.Portfolio.report)
607 file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) 645 file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
608 db_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) 646 db_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
647 redis_report.assert_not_called()
609 648
610 def test_print_orders(self): 649 report.reset_mock()
611 m = market.Market(self.ccxt, self.market_args()) 650 m = market.Market(self.ccxt, self.market_args(report_redis=False),
612 with mock.patch.object(m.report, "log_stage") as log_stage,\ 651 redis_config="redis_config", user_id=1)
613 mock.patch.object(m.balances, "fetch_balances") as fetch_balances,\ 652 with self.subTest(redis_config="present", report_redis=False),\
614 mock.patch.object(m, "prepare_trades") as prepare_trades,\ 653 mock.patch.object(m, "report") as report,\
615 mock.patch.object(m.trades, "prepare_orders") as prepare_orders: 654 mock.patch.object(m, "store_file_report") as file_report,\
616 m.print_orders() 655 mock.patch.object(m, "store_redis_report") as redis_report,\
656 mock.patch.object(m, "store_database_report") as db_report,\
657 mock.patch.object(market.datetime, "datetime") as time_mock:
617 658
618 log_stage.assert_called_with("print_orders") 659 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
619 fetch_balances.assert_called_with(tag="print_orders")
620 prepare_trades.assert_called_with(base_currency="BTC",
621 compute_value="average")
622 prepare_orders.assert_called_with(compute_value="average")
623 660
624 def test_print_balances(self): 661 m.store_report()
662 redis_report.assert_not_called()
663
664 report.reset_mock()
665 m = market.Market(self.ccxt, self.market_args(report_redis=True),
666 user_id=1)
667 with self.subTest(redis_config="absent", report_redis=True),\
668 mock.patch.object(m, "report") as report,\
669 mock.patch.object(m, "store_file_report") as file_report,\
670 mock.patch.object(m, "store_redis_report") as redis_report,\
671 mock.patch.object(m, "store_database_report") as db_report,\
672 mock.patch.object(market.datetime, "datetime") as time_mock:
673
674 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
675
676 m.store_report()
677 redis_report.assert_not_called()
678
679 report.reset_mock()
680 m = market.Market(self.ccxt, self.market_args(report_redis=True),
681 redis_config="redis_config", user_id=1)
682 with self.subTest(redis_config="present", report_redis=True),\
683 mock.patch.object(m, "report") as report,\
684 mock.patch.object(m, "store_file_report") as file_report,\
685 mock.patch.object(m, "store_redis_report") as redis_report,\
686 mock.patch.object(m, "store_database_report") as db_report,\
687 mock.patch.object(market.datetime, "datetime") as time_mock:
688
689 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
690
691 m.store_report()
692 redis_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
693
694 def test_print_tickers(self):
625 m = market.Market(self.ccxt, self.market_args()) 695 m = market.Market(self.ccxt, self.market_args())
626 696
627 with mock.patch.object(m.balances, "in_currency") as in_currency,\ 697 with mock.patch.object(m.balances, "in_currency") as in_currency,\
@@ -634,10 +704,8 @@ class MarketTest(WebMockTestCase):
634 "ETH": portfolio.Amount("BTC", "0.3"), 704 "ETH": portfolio.Amount("BTC", "0.3"),
635 } 705 }
636 706
637 m.print_balances() 707 m.print_tickers()
638 708
639 log_stage.assert_called_once_with("print_balances")
640 fetch_balances.assert_called_with()
641 print_log.assert_has_calls([ 709 print_log.assert_has_calls([
642 mock.call("total:"), 710 mock.call("total:"),
643 mock.call(portfolio.Amount("BTC", "0.95")), 711 mock.call(portfolio.Amount("BTC", "0.95")),
@@ -648,8 +716,8 @@ class MarketTest(WebMockTestCase):
648 @mock.patch("market.Market.store_report") 716 @mock.patch("market.Market.store_report")
649 def test_process(self, store_report, log_error, process): 717 def test_process(self, store_report, log_error, process):
650 m = market.Market(self.ccxt, self.market_args()) 718 m = market.Market(self.ccxt, self.market_args())
651 with self.subTest(before=False, after=False): 719 with self.subTest(actions=[], before=False, after=False):
652 m.process(None) 720 m.process([])
653 721
654 process.assert_not_called() 722 process.assert_not_called()
655 store_report.assert_called_once() 723 store_report.assert_called_once()
@@ -659,9 +727,9 @@ class MarketTest(WebMockTestCase):
659 log_error.reset_mock() 727 log_error.reset_mock()
660 store_report.reset_mock() 728 store_report.reset_mock()
661 with self.subTest(before=True, after=False): 729 with self.subTest(before=True, after=False):
662 m.process(None, before=True) 730 m.process(["foo"], before=True)
663 731
664 process.assert_called_once_with("sell_all", steps="before") 732 process.assert_called_once_with("foo", steps="before")
665 store_report.assert_called_once() 733 store_report.assert_called_once()
666 log_error.assert_not_called() 734 log_error.assert_not_called()
667 735
@@ -669,7 +737,7 @@ class MarketTest(WebMockTestCase):
669 log_error.reset_mock() 737 log_error.reset_mock()
670 store_report.reset_mock() 738 store_report.reset_mock()
671 with self.subTest(before=False, after=True): 739 with self.subTest(before=False, after=True):
672 m.process(None, after=True) 740 m.process(["sell_all"], after=True)
673 741
674 process.assert_called_once_with("sell_all", steps="after") 742 process.assert_called_once_with("sell_all", steps="after")
675 store_report.assert_called_once() 743 store_report.assert_called_once()
@@ -678,54 +746,30 @@ class MarketTest(WebMockTestCase):
678 process.reset_mock() 746 process.reset_mock()
679 log_error.reset_mock() 747 log_error.reset_mock()
680 store_report.reset_mock() 748 store_report.reset_mock()
681 with self.subTest(before=True, after=True): 749 with self.subTest(before=False, after=False):
682 m.process(None, before=True, after=True) 750 m.process(["foo"])
683 751
684 process.assert_has_calls([ 752 process.assert_called_once_with("foo", steps="all")
685 mock.call("sell_all", steps="before"),
686 mock.call("sell_all", steps="after"),
687 ])
688 store_report.assert_called_once() 753 store_report.assert_called_once()
689 log_error.assert_not_called() 754 log_error.assert_not_called()
690 755
691 process.reset_mock() 756 process.reset_mock()
692 log_error.reset_mock() 757 log_error.reset_mock()
693 store_report.reset_mock() 758 store_report.reset_mock()
694 with self.subTest(action="print_balances"),\ 759 with self.subTest(before=True, after=True):
695 mock.patch.object(m, "print_balances") as print_balances: 760 m.process(["sell_all"], before=True, after=True)
696 m.process(["print_balances"])
697 761
698 process.assert_not_called() 762 process.assert_called_once_with("sell_all", steps="all")
699 log_error.assert_not_called()
700 store_report.assert_called_once() 763 store_report.assert_called_once()
701 print_balances.assert_called_once_with()
702
703 log_error.reset_mock()
704 store_report.reset_mock()
705 with self.subTest(action="print_orders"),\
706 mock.patch.object(m, "print_orders") as print_orders,\
707 mock.patch.object(m, "print_balances") as print_balances:
708 m.process(["print_orders", "print_balances"])
709
710 process.assert_not_called()
711 log_error.assert_not_called() 764 log_error.assert_not_called()
712 store_report.assert_called_once()
713 print_orders.assert_called_once_with()
714 print_balances.assert_called_once_with()
715
716 log_error.reset_mock()
717 store_report.reset_mock()
718 with self.subTest(action="unknown"):
719 m.process(["unknown"])
720 log_error.assert_called_once_with("market_process", message="Unknown action unknown")
721 store_report.assert_called_once()
722 765
766 process.reset_mock()
723 log_error.reset_mock() 767 log_error.reset_mock()
724 store_report.reset_mock() 768 store_report.reset_mock()
725 with self.subTest(unhandled_exception=True): 769 with self.subTest(unhandled_exception=True):
726 process.side_effect = Exception("bouh") 770 process.side_effect = Exception("bouh")
727 771
728 m.process(None, before=True) 772 m.process(["some_action"], before=True)
729 log_error.assert_called_with("market_process", exception=mock.ANY) 773 log_error.assert_called_with("market_process", exception=mock.ANY)
730 store_report.assert_called_once() 774 store_report.assert_called_once()
731 775
@@ -768,24 +812,39 @@ class ProcessorTest(WebMockTestCase):
768 with self.assertRaises(TypeError): 812 with self.assertRaises(TypeError):
769 processor.select_steps(scenario, ["wait"]) 813 processor.select_steps(scenario, ["wait"])
770 814
815 def test_can_process(self):
816 processor = market.Processor(self.m)
817
818 with self.subTest(True):
819 self.assertTrue(processor.can_process("sell_all"))
820
821 with self.subTest(False):
822 self.assertFalse(processor.can_process("unknown_action"))
823
771 @mock.patch("market.Processor.process_step") 824 @mock.patch("market.Processor.process_step")
772 def test_process(self, process_step): 825 def test_process(self, process_step):
773 processor = market.Processor(self.m) 826 with self.subTest("unknown action"):
827 processor = market.Processor(self.m)
828 with self.assertRaises(TypeError):
829 processor.process("unknown_action")
774 830
775 processor.process("sell_all", foo="bar") 831 with self.subTest("nominal case"):
776 self.assertEqual(3, process_step.call_count) 832 processor = market.Processor(self.m)
777 833
778 steps = list(map(lambda x: x[1][1]["name"], process_step.mock_calls)) 834 processor.process("sell_all", foo="bar")
779 scenario_names = list(map(lambda x: x[1][0], process_step.mock_calls)) 835 self.assertEqual(3, process_step.call_count)
780 kwargs = list(map(lambda x: x[1][2], process_step.mock_calls))
781 self.assertEqual(["all_sell", "wait", "all_buy"], steps)
782 self.assertEqual(["sell_all", "sell_all", "sell_all"], scenario_names)
783 self.assertEqual([{"foo":"bar"}, {"foo":"bar"}, {"foo":"bar"}], kwargs)
784 836
785 process_step.reset_mock() 837 steps = list(map(lambda x: x[1][1]["name"], process_step.mock_calls))
838 scenario_names = list(map(lambda x: x[1][0], process_step.mock_calls))
839 kwargs = list(map(lambda x: x[1][2], process_step.mock_calls))
840 self.assertEqual(["all_sell", "wait", "all_buy"], steps)
841 self.assertEqual(["sell_all", "sell_all", "sell_all"], scenario_names)
842 self.assertEqual([{"foo":"bar"}, {"foo":"bar"}, {"foo":"bar"}], kwargs)
786 843
787 processor.process("sell_needed", steps=["before", "after"]) 844 process_step.reset_mock()
788 self.assertEqual(3, process_step.call_count) 845
846 processor.process("sell_needed", steps=["before", "after"])
847 self.assertEqual(3, process_step.call_count)
789 848
790 def test_method_arguments(self): 849 def test_method_arguments(self):
791 ccxt = mock.Mock(spec=market.ccxt.poloniexE) 850 ccxt = mock.Mock(spec=market.ccxt.poloniexE)
@@ -816,6 +875,9 @@ class ProcessorTest(WebMockTestCase):
816 method, arguments = processor.method_arguments("close_trades") 875 method, arguments = processor.method_arguments("close_trades")
817 self.assertEqual(m.trades.close_trades, method) 876 self.assertEqual(m.trades.close_trades, method)
818 877
878 method, arguments = processor.method_arguments("print_tickers")
879 self.assertEqual(m.print_tickers, method)
880
819 def test_process_step(self): 881 def test_process_step(self):
820 processor = market.Processor(self.m) 882 processor = market.Processor(self.m)
821 883
diff --git a/tests/test_store.py b/tests/test_store.py
index ffd2645..df113b7 100644
--- a/tests/test_store.py
+++ b/tests/test_store.py
@@ -459,6 +459,13 @@ class ReportStoreTest(WebMockTestCase):
459 459
460 self.assertEqual({"foo": "bar", "date": mock.ANY, "user_id": None, "market_id": None}, result) 460 self.assertEqual({"foo": "bar", "date": mock.ANY, "user_id": None, "market_id": None}, result)
461 461
462 def test_add_redis_status(self):
463 report_store = market.ReportStore(self.m)
464 result = report_store.add_redis_status({"foo": "bar"})
465
466 self.assertEqual({"foo": "bar"}, result)
467 self.assertEqual(result, report_store.redis_status[0])
468
462 def test_set_verbose(self): 469 def test_set_verbose(self):
463 report_store = market.ReportStore(self.m) 470 report_store = market.ReportStore(self.m)
464 with self.subTest(verbose=True): 471 with self.subTest(verbose=True):
@@ -534,6 +541,20 @@ class ReportStoreTest(WebMockTestCase):
534 self.assertEqual(("date1", "type1", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[0]) 541 self.assertEqual(("date1", "type1", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[0])
535 self.assertEqual(("date2", "type2", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[1]) 542 self.assertEqual(("date2", "type2", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[1])
536 543
544 def test_to_json_redis(self):
545 report_store = market.ReportStore(self.m)
546 report_store.redis_status.append({
547 "type": "type1", "foo": "bar", "bla": "bla"
548 })
549 report_store.redis_status.append({
550 "type": "type2", "foo": "bar", "bla": "bla"
551 })
552 logs = list(report_store.to_json_redis())
553
554 self.assertEqual(2, len(logs))
555 self.assertEqual(("type1", '{"foo": "bar", "bla": "bla"}'), logs[0])
556 self.assertEqual(("type2", '{"foo": "bar", "bla": "bla"}'), logs[1])
557
537 @mock.patch.object(market.ReportStore, "print_log") 558 @mock.patch.object(market.ReportStore, "print_log")
538 @mock.patch.object(market.ReportStore, "add_log") 559 @mock.patch.object(market.ReportStore, "add_log")
539 def test_log_stage(self, add_log, print_log): 560 def test_log_stage(self, add_log, print_log):
@@ -559,7 +580,8 @@ class ReportStoreTest(WebMockTestCase):
559 580
560 @mock.patch.object(market.ReportStore, "print_log") 581 @mock.patch.object(market.ReportStore, "print_log")
561 @mock.patch.object(market.ReportStore, "add_log") 582 @mock.patch.object(market.ReportStore, "add_log")
562 def test_log_balances(self, add_log, print_log): 583 @mock.patch.object(market.ReportStore, "add_redis_status")
584 def test_log_balances(self, add_redis_status, add_log, print_log):
563 report_store = market.ReportStore(self.m) 585 report_store = market.ReportStore(self.m)
564 self.m.balances.as_json.return_value = "json" 586 self.m.balances.as_json.return_value = "json"
565 self.m.balances.all = { "FOO": "bar", "BAR": "baz" } 587 self.m.balances.all = { "FOO": "bar", "BAR": "baz" }
@@ -575,10 +597,16 @@ class ReportStoreTest(WebMockTestCase):
575 'balances': 'json', 597 'balances': 'json',
576 'tag': 'tag' 598 'tag': 'tag'
577 }) 599 })
600 add_redis_status.assert_called_once_with({
601 'type': 'balance',
602 'balances': 'json',
603 'tag': 'tag'
604 })
578 605
579 @mock.patch.object(market.ReportStore, "print_log") 606 @mock.patch.object(market.ReportStore, "print_log")
580 @mock.patch.object(market.ReportStore, "add_log") 607 @mock.patch.object(market.ReportStore, "add_log")
581 def test_log_tickers(self, add_log, print_log): 608 @mock.patch.object(market.ReportStore, "add_redis_status")
609 def test_log_tickers(self, add_redis_status, add_log, print_log):
582 report_store = market.ReportStore(self.m) 610 report_store = market.ReportStore(self.m)
583 amounts = { 611 amounts = {
584 "BTC": portfolio.Amount("BTC", 10), 612 "BTC": portfolio.Amount("BTC", 10),
@@ -603,6 +631,21 @@ class ReportStoreTest(WebMockTestCase):
603 }, 631 },
604 'total': D('10.3') 632 'total': D('10.3')
605 }) 633 })
634 add_redis_status.assert_called_once_with({
635 'type': 'tickers',
636 'compute_value': 'default',
637 'balance_type': 'total',
638 'currency': 'BTC',
639 'balances': {
640 'BTC': D('10'),
641 'ETH': D('0.3')
642 },
643 'rates': {
644 'BTC': None,
645 'ETH': D('0.1')
646 },
647 'total': D('10.3')
648 })
606 649
607 add_log.reset_mock() 650 add_log.reset_mock()
608 compute_value = lambda x: x["bid"] 651 compute_value = lambda x: x["bid"]