diff options
author | Ismaël Bouya <ismael.bouya@normalesup.org> | 2018-04-20 20:20:02 +0200 |
---|---|---|
committer | Ismaël Bouya <ismael.bouya@normalesup.org> | 2018-04-21 00:42:16 +0200 |
commit | 1593c7a9f58ffaea8933f30f683f67c2b155f6b2 (patch) | |
tree | 1191b8438f1f7d52aa7ef736ec2def0595e311d0 | |
parent | ceb7fc4c9e76857fefbe1dfe3f4dd3830d065a6f (diff) | |
download | Trader-1593c7a9f58ffaea8933f30f683f67c2b155f6b2.tar.gz Trader-1593c7a9f58ffaea8933f30f683f67c2b155f6b2.tar.zst Trader-1593c7a9f58ffaea8933f30f683f67c2b155f6b2.zip |
Store some information to redis
-rw-r--r-- | main.py | 33 | ||||
-rw-r--r-- | market.py | 16 | ||||
-rw-r--r-- | requirements.txt | 1 | ||||
-rw-r--r-- | store.py | 46 | ||||
-rw-r--r-- | tests/test_main.py | 53 | ||||
-rw-r--r-- | tests/test_market.py | 84 | ||||
-rw-r--r-- | tests/test_store.py | 47 |
7 files changed, 245 insertions, 35 deletions
@@ -95,13 +95,25 @@ def parse_config(args): | |||
95 | del(args.db_password) | 95 | del(args.db_password) |
96 | del(args.db_database) | 96 | del(args.db_database) |
97 | 97 | ||
98 | redis_config = { | ||
99 | "host": args.redis_host, | ||
100 | "port": args.redis_port, | ||
101 | "db": args.redis_database, | ||
102 | } | ||
103 | if redis_config["host"].startswith("/"): | ||
104 | redis_config["unix_socket_path"] = redis_config.pop("host") | ||
105 | del(redis_config["port"]) | ||
106 | del(args.redis_host) | ||
107 | del(args.redis_port) | ||
108 | del(args.redis_database) | ||
109 | |||
98 | report_path = args.report_path | 110 | report_path = args.report_path |
99 | 111 | ||
100 | if report_path is not None and not \ | 112 | if report_path is not None and not \ |
101 | os.path.exists(report_path): | 113 | os.path.exists(report_path): |
102 | os.makedirs(report_path) | 114 | os.makedirs(report_path) |
103 | 115 | ||
104 | return pg_config | 116 | return pg_config, redis_config |
105 | 117 | ||
106 | def parse_args(argv): | 118 | def parse_args(argv): |
107 | parser = configargparse.ArgumentParser( | 119 | parser = configargparse.ArgumentParser( |
@@ -134,6 +146,10 @@ def parse_args(argv): | |||
134 | help="Store report to database (default)") | 146 | help="Store report to database (default)") |
135 | parser.add_argument("--no-report-db", action='store_false', dest="report_db", | 147 | parser.add_argument("--no-report-db", action='store_false', dest="report_db", |
136 | help="Don't store report to database") | 148 | help="Don't store report to database") |
149 | parser.add_argument("--report-redis", action='store_true', default=False, dest="report_redis", | ||
150 | help="Store report to redis") | ||
151 | parser.add_argument("--no-report-redis", action='store_false', dest="report_redis", | ||
152 | help="Don't store report to redis (default)") | ||
137 | parser.add_argument("--report-path", required=False, | 153 | parser.add_argument("--report-path", required=False, |
138 | help="Where to store the reports (default: absent, don't store)") | 154 | help="Where to store the reports (default: absent, don't store)") |
139 | parser.add_argument("--no-report-path", action='store_const', dest='report_path', const=None, | 155 | parser.add_argument("--no-report-path", action='store_const', dest='report_path', const=None, |
@@ -148,17 +164,24 @@ def parse_args(argv): | |||
148 | help="Password access to database (default: cryptoportfolio)") | 164 | help="Password access to database (default: cryptoportfolio)") |
149 | parser.add_argument("--db-database", default="cryptoportfolio", | 165 | parser.add_argument("--db-database", default="cryptoportfolio", |
150 | help="Database access to database (default: cryptoportfolio)") | 166 | help="Database access to database (default: cryptoportfolio)") |
167 | parser.add_argument("--redis-host", default="localhost", | ||
168 | help="Host access to database (default: localhost). Use path for socket") | ||
169 | parser.add_argument("--redis-port", default=6379, | ||
170 | help="Port access to redis (default: 6379)") | ||
171 | parser.add_argument("--redis-database", default=0, | ||
172 | help="Redis database to use (default: 0)") | ||
151 | 173 | ||
152 | parsed = parser.parse_args(argv) | 174 | parsed = parser.parse_args(argv) |
153 | if parsed.action is None: | 175 | if parsed.action is None: |
154 | parsed.action = ["sell_all"] | 176 | parsed.action = ["sell_all"] |
155 | return parsed | 177 | return parsed |
156 | 178 | ||
157 | def process(market_config, market_id, user_id, args, pg_config): | 179 | def process(market_config, market_id, user_id, args, pg_config, redis_config): |
158 | try: | 180 | try: |
159 | market.Market\ | 181 | market.Market\ |
160 | .from_config(market_config, args, market_id=market_id, | 182 | .from_config(market_config, args, market_id=market_id, |
161 | pg_config=pg_config, user_id=user_id)\ | 183 | pg_config=pg_config, redis_config=redis_config, |
184 | user_id=user_id)\ | ||
162 | .process(args.action, before=args.before, after=args.after) | 185 | .process(args.action, before=args.before, after=args.after) |
163 | except Exception as e: | 186 | except Exception as e: |
164 | print("{}: {}".format(e.__class__.__name__, e)) | 187 | print("{}: {}".format(e.__class__.__name__, e)) |
@@ -166,7 +189,7 @@ def process(market_config, market_id, user_id, args, pg_config): | |||
166 | def main(argv): | 189 | def main(argv): |
167 | args = parse_args(argv) | 190 | args = parse_args(argv) |
168 | 191 | ||
169 | pg_config = parse_config(args) | 192 | pg_config, redis_config = parse_config(args) |
170 | 193 | ||
171 | market.Portfolio.report.set_verbose(not args.quiet) | 194 | market.Portfolio.report.set_verbose(not args.quiet) |
172 | 195 | ||
@@ -183,7 +206,7 @@ def main(argv): | |||
183 | process_ = process | 206 | process_ = process |
184 | 207 | ||
185 | for market_id, market_config, user_id in fetch_markets(pg_config, args.user): | 208 | for market_id, market_config, user_id in fetch_markets(pg_config, args.user): |
186 | process_(market_config, market_id, user_id, args, pg_config) | 209 | process_(market_config, market_id, user_id, args, pg_config, redis_config) |
187 | 210 | ||
188 | if args.parallel: | 211 | if args.parallel: |
189 | for thread in threads: | 212 | for thread in threads: |
@@ -2,6 +2,7 @@ from ccxt import ExchangeError, NotSupported, RequestTimeout, InvalidNonce | |||
2 | import ccxt_wrapper as ccxt | 2 | import ccxt_wrapper as ccxt |
3 | import time | 3 | import time |
4 | import psycopg2 | 4 | import psycopg2 |
5 | import redis | ||
5 | from store import * | 6 | from store import * |
6 | from cachetools.func import ttl_cache | 7 | from cachetools.func import ttl_cache |
7 | from datetime import datetime | 8 | from datetime import datetime |
@@ -26,7 +27,7 @@ class Market: | |||
26 | self.balances = BalanceStore(self) | 27 | self.balances = BalanceStore(self) |
27 | self.processor = Processor(self) | 28 | self.processor = Processor(self) |
28 | 29 | ||
29 | for key in ["user_id", "market_id", "pg_config"]: | 30 | for key in ["user_id", "market_id", "pg_config", "redis_config"]: |
30 | setattr(self, key, kwargs.get(key, None)) | 31 | setattr(self, key, kwargs.get(key, None)) |
31 | 32 | ||
32 | self.report.log_market(self.args) | 33 | self.report.log_market(self.args) |
@@ -46,6 +47,8 @@ class Market: | |||
46 | self.store_file_report(date) | 47 | self.store_file_report(date) |
47 | if self.pg_config is not None and self.args.report_db: | 48 | if self.pg_config is not None and self.args.report_db: |
48 | self.store_database_report(date) | 49 | self.store_database_report(date) |
50 | if self.redis_config is not None and self.args.report_redis: | ||
51 | self.store_redis_report(date) | ||
49 | 52 | ||
50 | def store_file_report(self, date): | 53 | def store_file_report(self, date): |
51 | try: | 54 | try: |
@@ -74,6 +77,17 @@ class Market: | |||
74 | except Exception as e: | 77 | except Exception as e: |
75 | print("impossible to store report to database: {}; {}".format(e.__class__.__name__, e)) | 78 | print("impossible to store report to database: {}; {}".format(e.__class__.__name__, e)) |
76 | 79 | ||
80 | def store_redis_report(self, date): | ||
81 | try: | ||
82 | conn = redis.Redis(**self.redis_config) | ||
83 | for type_, log in self.report.to_json_redis(): | ||
84 | key = "/cryptoportfolio/{}/{}/{}".format(self.market_id, date.isoformat(), type_) | ||
85 | conn.set(key, log, ex=31*24*60*60) | ||
86 | key = "/cryptoportfolio/{}/latest/{}".format(self.market_id, type_) | ||
87 | conn.set(key, log) | ||
88 | except Exception as e: | ||
89 | print("impossible to store report to redis: {}; {}".format(e.__class__.__name__, e)) | ||
90 | |||
77 | def process(self, actions, before=False, after=False): | 91 | def process(self, actions, before=False, after=False): |
78 | try: | 92 | try: |
79 | for action in actions: | 93 | for action in actions: |
diff --git a/requirements.txt b/requirements.txt index 2451c80..3a4db2d 100644 --- a/requirements.txt +++ b/requirements.txt | |||
@@ -6,3 +6,4 @@ psycopg2==2.7.4 | |||
6 | retry==0.9.2 | 6 | retry==0.9.2 |
7 | cachetools==2.0.1 | 7 | cachetools==2.0.1 |
8 | configargparse==0.12.0 | 8 | configargparse==0.12.0 |
9 | redis==2.10.6 | ||
@@ -17,6 +17,7 @@ class ReportStore: | |||
17 | 17 | ||
18 | self.print_logs = [] | 18 | self.print_logs = [] |
19 | self.logs = [] | 19 | self.logs = [] |
20 | self.redis_status = [] | ||
20 | 21 | ||
21 | self.no_http_dup = no_http_dup | 22 | self.no_http_dup = no_http_dup |
22 | self.last_http = None | 23 | self.last_http = None |
@@ -46,6 +47,10 @@ class ReportStore: | |||
46 | self.logs.append(hash_) | 47 | self.logs.append(hash_) |
47 | return hash_ | 48 | return hash_ |
48 | 49 | ||
50 | def add_redis_status(self, hash_): | ||
51 | self.redis_status.append(hash_) | ||
52 | return hash_ | ||
53 | |||
49 | @staticmethod | 54 | @staticmethod |
50 | def default_json_serial(obj): | 55 | def default_json_serial(obj): |
51 | if isinstance(obj, (datetime.datetime, datetime.date)): | 56 | if isinstance(obj, (datetime.datetime, datetime.date)): |
@@ -63,6 +68,13 @@ class ReportStore: | |||
63 | json.dumps(log, default=self.default_json_serial, indent=" ") | 68 | json.dumps(log, default=self.default_json_serial, indent=" ") |
64 | ) | 69 | ) |
65 | 70 | ||
71 | def to_json_redis(self): | ||
72 | for log in (x.copy() for x in self.redis_status): | ||
73 | yield ( | ||
74 | log.pop("type"), | ||
75 | json.dumps(log, default=self.default_json_serial) | ||
76 | ) | ||
77 | |||
66 | def set_verbose(self, verbose_print): | 78 | def set_verbose(self, verbose_print): |
67 | self.verbose_print = verbose_print | 79 | self.verbose_print = verbose_print |
68 | 80 | ||
@@ -91,11 +103,14 @@ class ReportStore: | |||
91 | for currency, balance in self.market.balances.all.items(): | 103 | for currency, balance in self.market.balances.all.items(): |
92 | self.print_log("\t{}".format(balance)) | 104 | self.print_log("\t{}".format(balance)) |
93 | 105 | ||
94 | self.add_log({ | 106 | log = { |
95 | "type": "balance", | 107 | "type": "balance", |
96 | "tag": tag, | 108 | "tag": tag, |
97 | "balances": self.market.balances.as_json() | 109 | "balances": self.market.balances.as_json() |
98 | }) | 110 | } |
111 | |||
112 | self.add_log(log.copy()) | ||
113 | self.add_redis_status(log) | ||
99 | 114 | ||
100 | def log_tickers(self, amounts, other_currency, | 115 | def log_tickers(self, amounts, other_currency, |
101 | compute_value, type): | 116 | compute_value, type): |
@@ -107,15 +122,18 @@ class ReportStore: | |||
107 | for currency, amount in amounts.items(): | 122 | for currency, amount in amounts.items(): |
108 | values[currency] = amount.as_json()["value"] | 123 | values[currency] = amount.as_json()["value"] |
109 | rates[currency] = amount.rate | 124 | rates[currency] = amount.rate |
110 | self.add_log({ | 125 | log = { |
111 | "type": "tickers", | 126 | "type": "tickers", |
112 | "compute_value": compute_value, | 127 | "compute_value": compute_value, |
113 | "balance_type": type, | 128 | "balance_type": type, |
114 | "currency": other_currency, | 129 | "currency": other_currency, |
115 | "balances": values, | 130 | "balances": values, |
116 | "rates": rates, | 131 | "rates": rates, |
117 | "total": sum(amounts.values()).as_json()["value"] | 132 | "total": sum(amounts.values()).as_json()["value"] |
118 | }) | 133 | } |
134 | |||
135 | self.add_log(log.copy()) | ||
136 | self.add_redis_status(log) | ||
119 | 137 | ||
120 | def log_dispatch(self, amount, amounts, liquidity, repartition): | 138 | def log_dispatch(self, amount, amounts, liquidity, repartition): |
121 | self.add_log({ | 139 | self.add_log({ |
diff --git a/tests/test_main.py b/tests/test_main.py index d2f8029..b650870 100644 --- a/tests/test_main.py +++ b/tests/test_main.py | |||
@@ -135,16 +135,16 @@ class MainTest(WebMockTestCase): | |||
135 | args_mock.after = "after" | 135 | args_mock.after = "after" |
136 | self.assertEqual("", stdout_mock.getvalue()) | 136 | self.assertEqual("", stdout_mock.getvalue()) |
137 | 137 | ||
138 | main.process("config", 3, 1, args_mock, "pg_config") | 138 | main.process("config", 3, 1, args_mock, "pg_config", "redis_config") |
139 | 139 | ||
140 | market_mock.from_config.assert_has_calls([ | 140 | market_mock.from_config.assert_has_calls([ |
141 | mock.call("config", args_mock, pg_config="pg_config", market_id=3, user_id=1), | 141 | mock.call("config", args_mock, pg_config="pg_config", redis_config="redis_config", market_id=3, user_id=1), |
142 | mock.call().process("action", before="before", after="after"), | 142 | mock.call().process("action", before="before", after="after"), |
143 | ]) | 143 | ]) |
144 | 144 | ||
145 | with self.subTest(exception=True): | 145 | with self.subTest(exception=True): |
146 | market_mock.from_config.side_effect = Exception("boo") | 146 | market_mock.from_config.side_effect = Exception("boo") |
147 | main.process(3, "config", 1, args_mock, "pg_config") | 147 | main.process(3, "config", 1, args_mock, "pg_config", "redis_config") |
148 | self.assertEqual("Exception: boo\n", stdout_mock.getvalue()) | 148 | self.assertEqual("Exception: boo\n", stdout_mock.getvalue()) |
149 | 149 | ||
150 | def test_main(self): | 150 | def test_main(self): |
@@ -159,7 +159,7 @@ class MainTest(WebMockTestCase): | |||
159 | args_mock.user = "user" | 159 | args_mock.user = "user" |
160 | parse_args.return_value = args_mock | 160 | parse_args.return_value = args_mock |
161 | 161 | ||
162 | parse_config.return_value = "pg_config" | 162 | parse_config.return_value = ["pg_config", "redis_config"] |
163 | 163 | ||
164 | fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] | 164 | fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] |
165 | 165 | ||
@@ -171,8 +171,8 @@ class MainTest(WebMockTestCase): | |||
171 | 171 | ||
172 | self.assertEqual(2, process.call_count) | 172 | self.assertEqual(2, process.call_count) |
173 | process.assert_has_calls([ | 173 | process.assert_has_calls([ |
174 | mock.call("config1", 3, 1, args_mock, "pg_config"), | 174 | mock.call("config1", 3, 1, args_mock, "pg_config", "redis_config"), |
175 | mock.call("config2", 1, 2, args_mock, "pg_config"), | 175 | mock.call("config2", 1, 2, args_mock, "pg_config", "redis_config"), |
176 | ]) | 176 | ]) |
177 | with self.subTest(parallel=True): | 177 | with self.subTest(parallel=True): |
178 | with mock.patch("main.parse_args") as parse_args,\ | 178 | with mock.patch("main.parse_args") as parse_args,\ |
@@ -187,7 +187,7 @@ class MainTest(WebMockTestCase): | |||
187 | args_mock.user = "user" | 187 | args_mock.user = "user" |
188 | parse_args.return_value = args_mock | 188 | parse_args.return_value = args_mock |
189 | 189 | ||
190 | parse_config.return_value = "pg_config" | 190 | parse_config.return_value = ["pg_config", "redis_config"] |
191 | 191 | ||
192 | fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] | 192 | fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] |
193 | 193 | ||
@@ -202,9 +202,9 @@ class MainTest(WebMockTestCase): | |||
202 | self.assertEqual(2, process.call_count) | 202 | self.assertEqual(2, process.call_count) |
203 | process.assert_has_calls([ | 203 | process.assert_has_calls([ |
204 | mock.call.__bool__(), | 204 | mock.call.__bool__(), |
205 | mock.call("config1", 3, 1, args_mock, "pg_config"), | 205 | mock.call("config1", 3, 1, args_mock, "pg_config", "redis_config"), |
206 | mock.call.__bool__(), | 206 | mock.call.__bool__(), |
207 | mock.call("config2", 1, 2, args_mock, "pg_config"), | 207 | mock.call("config2", 1, 2, args_mock, "pg_config", "redis_config"), |
208 | ]) | 208 | ]) |
209 | with self.subTest(quiet=True): | 209 | with self.subTest(quiet=True): |
210 | with mock.patch("main.parse_args") as parse_args,\ | 210 | with mock.patch("main.parse_args") as parse_args,\ |
@@ -219,7 +219,7 @@ class MainTest(WebMockTestCase): | |||
219 | args_mock.user = "user" | 219 | args_mock.user = "user" |
220 | parse_args.return_value = args_mock | 220 | parse_args.return_value = args_mock |
221 | 221 | ||
222 | parse_config.return_value = "pg_config" | 222 | parse_config.return_value = ["pg_config", "redis_config"] |
223 | 223 | ||
224 | fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] | 224 | fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] |
225 | 225 | ||
@@ -240,7 +240,7 @@ class MainTest(WebMockTestCase): | |||
240 | args_mock.user = "user" | 240 | args_mock.user = "user" |
241 | parse_args.return_value = args_mock | 241 | parse_args.return_value = args_mock |
242 | 242 | ||
243 | parse_config.return_value = "pg_config" | 243 | parse_config.return_value = ["pg_config", "redis_config"] |
244 | 244 | ||
245 | fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] | 245 | fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]] |
246 | 246 | ||
@@ -259,15 +259,39 @@ class MainTest(WebMockTestCase): | |||
259 | "db_user": "user", | 259 | "db_user": "user", |
260 | "db_password": "password", | 260 | "db_password": "password", |
261 | "db_database": "database", | 261 | "db_database": "database", |
262 | "redis_host": "rhost", | ||
263 | "redis_port": "rport", | ||
264 | "redis_database": "rdb", | ||
262 | "report_path": None, | 265 | "report_path": None, |
263 | }) | 266 | }) |
264 | 267 | ||
265 | result = main.parse_config(args) | 268 | db_config, redis_config = main.parse_config(args) |
266 | self.assertEqual({ "host": "host", "port": "port", "user": | 269 | self.assertEqual({ "host": "host", "port": "port", "user": |
267 | "user", "password": "password", "database": "database" | 270 | "user", "password": "password", "database": "database" |
268 | }, result) | 271 | }, db_config) |
272 | self.assertEqual({ "host": "rhost", "port": "rport", "db": | ||
273 | "rdb"}, redis_config) | ||
274 | |||
269 | with self.assertRaises(AttributeError): | 275 | with self.assertRaises(AttributeError): |
270 | args.db_password | 276 | args.db_password |
277 | with self.assertRaises(AttributeError): | ||
278 | args.redis_host | ||
279 | |||
280 | with self.subTest(redis_host="socket"): | ||
281 | args = main.configargparse.Namespace(**{ | ||
282 | "db_host": "host", | ||
283 | "db_port": "port", | ||
284 | "db_user": "user", | ||
285 | "db_password": "password", | ||
286 | "db_database": "database", | ||
287 | "redis_host": "/run/foo", | ||
288 | "redis_port": "rport", | ||
289 | "redis_database": "rdb", | ||
290 | "report_path": None, | ||
291 | }) | ||
292 | |||
293 | db_config, redis_config = main.parse_config(args) | ||
294 | self.assertEqual({ "unix_socket_path": "/run/foo", "db": "rdb"}, redis_config) | ||
271 | 295 | ||
272 | with self.subTest(report_path="present"): | 296 | with self.subTest(report_path="present"): |
273 | args = main.configargparse.Namespace(**{ | 297 | args = main.configargparse.Namespace(**{ |
@@ -276,6 +300,9 @@ class MainTest(WebMockTestCase): | |||
276 | "db_user": "user", | 300 | "db_user": "user", |
277 | "db_password": "password", | 301 | "db_password": "password", |
278 | "db_database": "database", | 302 | "db_database": "database", |
303 | "redis_host": "rhost", | ||
304 | "redis_port": "rport", | ||
305 | "redis_database": "rdb", | ||
279 | "report_path": "report_path", | 306 | "report_path": "report_path", |
280 | }) | 307 | }) |
281 | 308 | ||
diff --git a/tests/test_market.py b/tests/test_market.py index b41cd6a..e3482b8 100644 --- a/tests/test_market.py +++ b/tests/test_market.py | |||
@@ -530,23 +530,55 @@ class MarketTest(WebMockTestCase): | |||
530 | m.store_database_report(datetime.datetime(2018, 3, 24)) | 530 | m.store_database_report(datetime.datetime(2018, 3, 24)) |
531 | self.assertEqual(stdout_mock.getvalue(), "impossible to store report to database: Exception; Bouh\n") | 531 | self.assertEqual(stdout_mock.getvalue(), "impossible to store report to database: Exception; Bouh\n") |
532 | 532 | ||
533 | @mock.patch.object(market, "redis") | ||
534 | def test_store_redis_report(self, redis): | ||
535 | connect_mock = mock.Mock() | ||
536 | redis.Redis.return_value = connect_mock | ||
537 | |||
538 | m = market.Market(self.ccxt, self.market_args(), | ||
539 | redis_config={"config": "redis_config"}, market_id=1) | ||
540 | |||
541 | with self.subTest(error=False),\ | ||
542 | mock.patch.object(m, "report") as report: | ||
543 | report.to_json_redis.return_value = [ | ||
544 | ("type1", "payload1"), | ||
545 | ("type2", "payload2"), | ||
546 | ] | ||
547 | m.store_redis_report(datetime.datetime(2018, 3, 24)) | ||
548 | connect_mock.assert_has_calls([ | ||
549 | mock.call.set("/cryptoportfolio/1/2018-03-24T00:00:00/type1", "payload1", ex=31*24*60*60), | ||
550 | mock.call.set("/cryptoportfolio/1/latest/type1", "payload1"), | ||
551 | mock.call.set("/cryptoportfolio/1/2018-03-24T00:00:00/type2", "payload2", ex=31*24*60*60), | ||
552 | mock.call.set("/cryptoportfolio/1/latest/type2", "payload2"), | ||
553 | ]) | ||
554 | |||
555 | connect_mock.reset_mock() | ||
556 | with self.subTest(error=True),\ | ||
557 | mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock: | ||
558 | redis.Redis.side_effect = Exception("Bouh") | ||
559 | m.store_redis_report(datetime.datetime(2018, 3, 24)) | ||
560 | self.assertEqual(stdout_mock.getvalue(), "impossible to store report to redis: Exception; Bouh\n") | ||
561 | |||
533 | def test_store_report(self): | 562 | def test_store_report(self): |
534 | m = market.Market(self.ccxt, self.market_args(report_db=False), user_id=1) | 563 | m = market.Market(self.ccxt, self.market_args(report_db=False), user_id=1) |
535 | with self.subTest(file=None, pg_config=None),\ | 564 | with self.subTest(file=None, pg_config=None),\ |
536 | mock.patch.object(m, "report") as report,\ | 565 | mock.patch.object(m, "report") as report,\ |
537 | mock.patch.object(m, "store_database_report") as db_report,\ | 566 | mock.patch.object(m, "store_database_report") as db_report,\ |
567 | mock.patch.object(m, "store_redis_report") as redis_report,\ | ||
538 | mock.patch.object(m, "store_file_report") as file_report: | 568 | mock.patch.object(m, "store_file_report") as file_report: |
539 | m.store_report() | 569 | m.store_report() |
540 | report.merge.assert_called_with(store.Portfolio.report) | 570 | report.merge.assert_called_with(store.Portfolio.report) |
541 | 571 | ||
542 | file_report.assert_not_called() | 572 | file_report.assert_not_called() |
543 | db_report.assert_not_called() | 573 | db_report.assert_not_called() |
574 | redis_report.assert_not_called() | ||
544 | 575 | ||
545 | report.reset_mock() | 576 | report.reset_mock() |
546 | m = market.Market(self.ccxt, self.market_args(report_db=False, report_path="present"), user_id=1) | 577 | m = market.Market(self.ccxt, self.market_args(report_db=False, report_path="present"), user_id=1) |
547 | with self.subTest(file="present", pg_config=None),\ | 578 | with self.subTest(file="present", pg_config=None),\ |
548 | mock.patch.object(m, "report") as report,\ | 579 | mock.patch.object(m, "report") as report,\ |
549 | mock.patch.object(m, "store_file_report") as file_report,\ | 580 | mock.patch.object(m, "store_file_report") as file_report,\ |
581 | mock.patch.object(m, "store_redis_report") as redis_report,\ | ||
550 | mock.patch.object(m, "store_database_report") as db_report,\ | 582 | mock.patch.object(m, "store_database_report") as db_report,\ |
551 | mock.patch.object(market.datetime, "datetime") as time_mock: | 583 | mock.patch.object(market.datetime, "datetime") as time_mock: |
552 | 584 | ||
@@ -557,12 +589,14 @@ class MarketTest(WebMockTestCase): | |||
557 | report.merge.assert_called_with(store.Portfolio.report) | 589 | report.merge.assert_called_with(store.Portfolio.report) |
558 | file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) | 590 | file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) |
559 | db_report.assert_not_called() | 591 | db_report.assert_not_called() |
592 | redis_report.assert_not_called() | ||
560 | 593 | ||
561 | report.reset_mock() | 594 | report.reset_mock() |
562 | m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), user_id=1) | 595 | m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), user_id=1) |
563 | with self.subTest(file="present", pg_config=None, report_db=True),\ | 596 | with self.subTest(file="present", pg_config=None, report_db=True),\ |
564 | mock.patch.object(m, "report") as report,\ | 597 | mock.patch.object(m, "report") as report,\ |
565 | mock.patch.object(m, "store_file_report") as file_report,\ | 598 | mock.patch.object(m, "store_file_report") as file_report,\ |
599 | mock.patch.object(m, "store_redis_report") as redis_report,\ | ||
566 | mock.patch.object(m, "store_database_report") as db_report,\ | 600 | mock.patch.object(m, "store_database_report") as db_report,\ |
567 | mock.patch.object(market.datetime, "datetime") as time_mock: | 601 | mock.patch.object(market.datetime, "datetime") as time_mock: |
568 | 602 | ||
@@ -573,12 +607,14 @@ class MarketTest(WebMockTestCase): | |||
573 | report.merge.assert_called_with(store.Portfolio.report) | 607 | report.merge.assert_called_with(store.Portfolio.report) |
574 | file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) | 608 | file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) |
575 | db_report.assert_not_called() | 609 | db_report.assert_not_called() |
610 | redis_report.assert_not_called() | ||
576 | 611 | ||
577 | report.reset_mock() | 612 | report.reset_mock() |
578 | m = market.Market(self.ccxt, self.market_args(report_db=True), pg_config="present", user_id=1) | 613 | m = market.Market(self.ccxt, self.market_args(report_db=True), pg_config="present", user_id=1) |
579 | with self.subTest(file=None, pg_config="present"),\ | 614 | with self.subTest(file=None, pg_config="present"),\ |
580 | mock.patch.object(m, "report") as report,\ | 615 | mock.patch.object(m, "report") as report,\ |
581 | mock.patch.object(m, "store_file_report") as file_report,\ | 616 | mock.patch.object(m, "store_file_report") as file_report,\ |
617 | mock.patch.object(m, "store_redis_report") as redis_report,\ | ||
582 | mock.patch.object(m, "store_database_report") as db_report,\ | 618 | mock.patch.object(m, "store_database_report") as db_report,\ |
583 | mock.patch.object(market.datetime, "datetime") as time_mock: | 619 | mock.patch.object(market.datetime, "datetime") as time_mock: |
584 | 620 | ||
@@ -589,6 +625,7 @@ class MarketTest(WebMockTestCase): | |||
589 | report.merge.assert_called_with(store.Portfolio.report) | 625 | report.merge.assert_called_with(store.Portfolio.report) |
590 | file_report.assert_not_called() | 626 | file_report.assert_not_called() |
591 | db_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) | 627 | db_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) |
628 | redis_report.assert_not_called() | ||
592 | 629 | ||
593 | report.reset_mock() | 630 | report.reset_mock() |
594 | m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), | 631 | m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), |
@@ -596,6 +633,7 @@ class MarketTest(WebMockTestCase): | |||
596 | with self.subTest(file="present", pg_config="present"),\ | 633 | with self.subTest(file="present", pg_config="present"),\ |
597 | mock.patch.object(m, "report") as report,\ | 634 | mock.patch.object(m, "report") as report,\ |
598 | mock.patch.object(m, "store_file_report") as file_report,\ | 635 | mock.patch.object(m, "store_file_report") as file_report,\ |
636 | mock.patch.object(m, "store_redis_report") as redis_report,\ | ||
599 | mock.patch.object(m, "store_database_report") as db_report,\ | 637 | mock.patch.object(m, "store_database_report") as db_report,\ |
600 | mock.patch.object(market.datetime, "datetime") as time_mock: | 638 | mock.patch.object(market.datetime, "datetime") as time_mock: |
601 | 639 | ||
@@ -606,6 +644,52 @@ class MarketTest(WebMockTestCase): | |||
606 | report.merge.assert_called_with(store.Portfolio.report) | 644 | report.merge.assert_called_with(store.Portfolio.report) |
607 | file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) | 645 | file_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) |
608 | db_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) | 646 | db_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) |
647 | redis_report.assert_not_called() | ||
648 | |||
649 | report.reset_mock() | ||
650 | m = market.Market(self.ccxt, self.market_args(report_redis=False), | ||
651 | redis_config="redis_config", user_id=1) | ||
652 | with self.subTest(redis_config="present", report_redis=False),\ | ||
653 | mock.patch.object(m, "report") as report,\ | ||
654 | mock.patch.object(m, "store_file_report") as file_report,\ | ||
655 | mock.patch.object(m, "store_redis_report") as redis_report,\ | ||
656 | mock.patch.object(m, "store_database_report") as db_report,\ | ||
657 | mock.patch.object(market.datetime, "datetime") as time_mock: | ||
658 | |||
659 | time_mock.now.return_value = datetime.datetime(2018, 2, 25) | ||
660 | |||
661 | m.store_report() | ||
662 | redis_report.assert_not_called() | ||
663 | |||
664 | report.reset_mock() | ||
665 | m = market.Market(self.ccxt, self.market_args(report_redis=True), | ||
666 | user_id=1) | ||
667 | with self.subTest(redis_config="absent", report_redis=True),\ | ||
668 | mock.patch.object(m, "report") as report,\ | ||
669 | mock.patch.object(m, "store_file_report") as file_report,\ | ||
670 | mock.patch.object(m, "store_redis_report") as redis_report,\ | ||
671 | mock.patch.object(m, "store_database_report") as db_report,\ | ||
672 | mock.patch.object(market.datetime, "datetime") as time_mock: | ||
673 | |||
674 | time_mock.now.return_value = datetime.datetime(2018, 2, 25) | ||
675 | |||
676 | m.store_report() | ||
677 | redis_report.assert_not_called() | ||
678 | |||
679 | report.reset_mock() | ||
680 | m = market.Market(self.ccxt, self.market_args(report_redis=True), | ||
681 | redis_config="redis_config", user_id=1) | ||
682 | with self.subTest(redis_config="present", report_redis=True),\ | ||
683 | mock.patch.object(m, "report") as report,\ | ||
684 | mock.patch.object(m, "store_file_report") as file_report,\ | ||
685 | mock.patch.object(m, "store_redis_report") as redis_report,\ | ||
686 | mock.patch.object(m, "store_database_report") as db_report,\ | ||
687 | mock.patch.object(market.datetime, "datetime") as time_mock: | ||
688 | |||
689 | time_mock.now.return_value = datetime.datetime(2018, 2, 25) | ||
690 | |||
691 | m.store_report() | ||
692 | redis_report.assert_called_once_with(datetime.datetime(2018, 2, 25)) | ||
609 | 693 | ||
610 | def test_print_tickers(self): | 694 | def test_print_tickers(self): |
611 | m = market.Market(self.ccxt, self.market_args()) | 695 | m = market.Market(self.ccxt, self.market_args()) |
diff --git a/tests/test_store.py b/tests/test_store.py index ffd2645..df113b7 100644 --- a/tests/test_store.py +++ b/tests/test_store.py | |||
@@ -459,6 +459,13 @@ class ReportStoreTest(WebMockTestCase): | |||
459 | 459 | ||
460 | self.assertEqual({"foo": "bar", "date": mock.ANY, "user_id": None, "market_id": None}, result) | 460 | self.assertEqual({"foo": "bar", "date": mock.ANY, "user_id": None, "market_id": None}, result) |
461 | 461 | ||
462 | def test_add_redis_status(self): | ||
463 | report_store = market.ReportStore(self.m) | ||
464 | result = report_store.add_redis_status({"foo": "bar"}) | ||
465 | |||
466 | self.assertEqual({"foo": "bar"}, result) | ||
467 | self.assertEqual(result, report_store.redis_status[0]) | ||
468 | |||
462 | def test_set_verbose(self): | 469 | def test_set_verbose(self): |
463 | report_store = market.ReportStore(self.m) | 470 | report_store = market.ReportStore(self.m) |
464 | with self.subTest(verbose=True): | 471 | with self.subTest(verbose=True): |
@@ -534,6 +541,20 @@ class ReportStoreTest(WebMockTestCase): | |||
534 | self.assertEqual(("date1", "type1", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[0]) | 541 | self.assertEqual(("date1", "type1", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[0]) |
535 | self.assertEqual(("date2", "type2", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[1]) | 542 | self.assertEqual(("date2", "type2", '{\n "foo": "bar",\n "bla": "bla"\n}'), logs[1]) |
536 | 543 | ||
544 | def test_to_json_redis(self): | ||
545 | report_store = market.ReportStore(self.m) | ||
546 | report_store.redis_status.append({ | ||
547 | "type": "type1", "foo": "bar", "bla": "bla" | ||
548 | }) | ||
549 | report_store.redis_status.append({ | ||
550 | "type": "type2", "foo": "bar", "bla": "bla" | ||
551 | }) | ||
552 | logs = list(report_store.to_json_redis()) | ||
553 | |||
554 | self.assertEqual(2, len(logs)) | ||
555 | self.assertEqual(("type1", '{"foo": "bar", "bla": "bla"}'), logs[0]) | ||
556 | self.assertEqual(("type2", '{"foo": "bar", "bla": "bla"}'), logs[1]) | ||
557 | |||
537 | @mock.patch.object(market.ReportStore, "print_log") | 558 | @mock.patch.object(market.ReportStore, "print_log") |
538 | @mock.patch.object(market.ReportStore, "add_log") | 559 | @mock.patch.object(market.ReportStore, "add_log") |
539 | def test_log_stage(self, add_log, print_log): | 560 | def test_log_stage(self, add_log, print_log): |
@@ -559,7 +580,8 @@ class ReportStoreTest(WebMockTestCase): | |||
559 | 580 | ||
560 | @mock.patch.object(market.ReportStore, "print_log") | 581 | @mock.patch.object(market.ReportStore, "print_log") |
561 | @mock.patch.object(market.ReportStore, "add_log") | 582 | @mock.patch.object(market.ReportStore, "add_log") |
562 | def test_log_balances(self, add_log, print_log): | 583 | @mock.patch.object(market.ReportStore, "add_redis_status") |
584 | def test_log_balances(self, add_redis_status, add_log, print_log): | ||
563 | report_store = market.ReportStore(self.m) | 585 | report_store = market.ReportStore(self.m) |
564 | self.m.balances.as_json.return_value = "json" | 586 | self.m.balances.as_json.return_value = "json" |
565 | self.m.balances.all = { "FOO": "bar", "BAR": "baz" } | 587 | self.m.balances.all = { "FOO": "bar", "BAR": "baz" } |
@@ -575,10 +597,16 @@ class ReportStoreTest(WebMockTestCase): | |||
575 | 'balances': 'json', | 597 | 'balances': 'json', |
576 | 'tag': 'tag' | 598 | 'tag': 'tag' |
577 | }) | 599 | }) |
600 | add_redis_status.assert_called_once_with({ | ||
601 | 'type': 'balance', | ||
602 | 'balances': 'json', | ||
603 | 'tag': 'tag' | ||
604 | }) | ||
578 | 605 | ||
579 | @mock.patch.object(market.ReportStore, "print_log") | 606 | @mock.patch.object(market.ReportStore, "print_log") |
580 | @mock.patch.object(market.ReportStore, "add_log") | 607 | @mock.patch.object(market.ReportStore, "add_log") |
581 | def test_log_tickers(self, add_log, print_log): | 608 | @mock.patch.object(market.ReportStore, "add_redis_status") |
609 | def test_log_tickers(self, add_redis_status, add_log, print_log): | ||
582 | report_store = market.ReportStore(self.m) | 610 | report_store = market.ReportStore(self.m) |
583 | amounts = { | 611 | amounts = { |
584 | "BTC": portfolio.Amount("BTC", 10), | 612 | "BTC": portfolio.Amount("BTC", 10), |
@@ -603,6 +631,21 @@ class ReportStoreTest(WebMockTestCase): | |||
603 | }, | 631 | }, |
604 | 'total': D('10.3') | 632 | 'total': D('10.3') |
605 | }) | 633 | }) |
634 | add_redis_status.assert_called_once_with({ | ||
635 | 'type': 'tickers', | ||
636 | 'compute_value': 'default', | ||
637 | 'balance_type': 'total', | ||
638 | 'currency': 'BTC', | ||
639 | 'balances': { | ||
640 | 'BTC': D('10'), | ||
641 | 'ETH': D('0.3') | ||
642 | }, | ||
643 | 'rates': { | ||
644 | 'BTC': None, | ||
645 | 'ETH': D('0.1') | ||
646 | }, | ||
647 | 'total': D('10.3') | ||
648 | }) | ||
606 | 649 | ||
607 | add_log.reset_mock() | 650 | add_log.reset_mock() |
608 | compute_value = lambda x: x["bid"] | 651 | compute_value = lambda x: x["bid"] |