aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--ccxt_wrapper.py2
-rw-r--r--main.py11
-rw-r--r--market.py5
-rw-r--r--portfolio.py6
-rw-r--r--store.py72
-rw-r--r--tests/test_ccxt_wrapper.py12
-rw-r--r--tests/test_main.py4
-rw-r--r--tests/test_market.py8
-rw-r--r--tests/test_portfolio.py4
-rw-r--r--tests/test_store.py167
10 files changed, 204 insertions, 87 deletions
diff --git a/ccxt_wrapper.py b/ccxt_wrapper.py
index bedf84b..366586c 100644
--- a/ccxt_wrapper.py
+++ b/ccxt_wrapper.py
@@ -47,6 +47,8 @@ class poloniexE(poloniex):
47 self.session._parent = self 47 self.session._parent = self
48 48
49 def request_wrap(self, *args, **kwargs): 49 def request_wrap(self, *args, **kwargs):
50 kwargs["headers"]["X-market-id"] = str(self._parent._market.market_id)
51 kwargs["headers"]["X-user-id"] = str(self._parent._market.user_id)
50 try: 52 try:
51 r = self.origin_request(*args, **kwargs) 53 r = self.origin_request(*args, **kwargs)
52 self._parent._market.report.log_http_request(args[0], 54 self._parent._market.report.log_http_request(args[0],
diff --git a/main.py b/main.py
index 6383ed1..2cfb01d 100644
--- a/main.py
+++ b/main.py
@@ -1,4 +1,3 @@
1from datetime import datetime
2import configargparse 1import configargparse
3import psycopg2 2import psycopg2
4import os 3import os
@@ -170,13 +169,21 @@ def main(argv):
170 import threading 169 import threading
171 market.Portfolio.start_worker() 170 market.Portfolio.start_worker()
172 171
172 threads = []
173 def process_(*args): 173 def process_(*args):
174 threading.Thread(target=process, args=args).start() 174 thread = threading.Thread(target=process, args=args)
175 thread.start()
176 threads.append(thread)
175 else: 177 else:
176 process_ = process 178 process_ = process
177 179
178 for market_id, market_config, user_id in fetch_markets(pg_config, args.user): 180 for market_id, market_config, user_id in fetch_markets(pg_config, args.user):
179 process_(market_config, market_id, user_id, args, pg_config) 181 process_(market_config, market_id, user_id, args, pg_config)
180 182
183 if args.parallel:
184 for thread in threads:
185 thread.join()
186 market.Portfolio.stop_worker()
187
181if __name__ == '__main__': # pragma: no cover 188if __name__ == '__main__': # pragma: no cover
182 main(sys.argv[1:]) 189 main(sys.argv[1:])
diff --git a/market.py b/market.py
index e16641c..7a37cf6 100644
--- a/market.py
+++ b/market.py
@@ -5,6 +5,7 @@ import psycopg2
5from store import * 5from store import *
6from cachetools.func import ttl_cache 6from cachetools.func import ttl_cache
7from datetime import datetime 7from datetime import datetime
8import datetime
8from retry import retry 9from retry import retry
9import portfolio 10import portfolio
10 11
@@ -28,7 +29,7 @@ class Market:
28 for key in ["user_id", "market_id", "pg_config"]: 29 for key in ["user_id", "market_id", "pg_config"]:
29 setattr(self, key, kwargs.get(key, None)) 30 setattr(self, key, kwargs.get(key, None))
30 31
31 self.report.log_market(self.args, self.user_id, self.market_id) 32 self.report.log_market(self.args)
32 33
33 @classmethod 34 @classmethod
34 def from_config(cls, config, args, **kwargs): 35 def from_config(cls, config, args, **kwargs):
@@ -40,7 +41,7 @@ class Market:
40 41
41 def store_report(self): 42 def store_report(self):
42 self.report.merge(Portfolio.report) 43 self.report.merge(Portfolio.report)
43 date = datetime.now() 44 date = datetime.datetime.now()
44 if self.args.report_path is not None: 45 if self.args.report_path is not None:
45 self.store_file_report(date) 46 self.store_file_report(date)
46 if self.pg_config is not None and self.args.report_db: 47 if self.pg_config is not None and self.args.report_db:
diff --git a/portfolio.py b/portfolio.py
index 535aaa8..146ee79 100644
--- a/portfolio.py
+++ b/portfolio.py
@@ -1,4 +1,4 @@
1from datetime import datetime 1import datetime
2from retry import retry 2from retry import retry
3from decimal import Decimal as D, ROUND_DOWN 3from decimal import Decimal as D, ROUND_DOWN
4from ccxt import ExchangeError, InsufficientFunds, ExchangeNotAvailable, InvalidOrder, OrderNotCached, OrderNotFound, RequestTimeout, InvalidNonce 4from ccxt import ExchangeError, InsufficientFunds, ExchangeNotAvailable, InvalidOrder, OrderNotCached, OrderNotFound, RequestTimeout, InvalidNonce
@@ -492,7 +492,7 @@ class Order:
492 self.market.report.log_debug_action(action) 492 self.market.report.log_debug_action(action)
493 self.results.append({"debug": True, "id": -1}) 493 self.results.append({"debug": True, "id": -1})
494 else: 494 else:
495 self.start_date = datetime.now() 495 self.start_date = datetime.datetime.now()
496 try: 496 try:
497 self.results.append(self.market.ccxt.create_order(symbol, 'limit', self.action, amount, price=self.rate, account=self.account)) 497 self.results.append(self.market.ccxt.create_order(symbol, 'limit', self.action, amount, price=self.rate, account=self.account))
498 except InvalidOrder: 498 except InvalidOrder:
@@ -677,7 +677,7 @@ class Mouvement:
677 self.action = hash_.get("type") 677 self.action = hash_.get("type")
678 self.fee_rate = D(hash_.get("fee", -1)) 678 self.fee_rate = D(hash_.get("fee", -1))
679 try: 679 try:
680 self.date = datetime.strptime(hash_.get("date", ""), '%Y-%m-%d %H:%M:%S') 680 self.date = datetime.datetime.strptime(hash_.get("date", ""), '%Y-%m-%d %H:%M:%S')
681 except ValueError: 681 except ValueError:
682 self.date = None 682 self.date = None
683 self.rate = D(hash_.get("rate", 0)) 683 self.rate = D(hash_.get("rate", 0))
diff --git a/store.py b/store.py
index 67e8a8f..467dd4b 100644
--- a/store.py
+++ b/store.py
@@ -3,7 +3,7 @@ import requests
3import portfolio 3import portfolio
4import simplejson as json 4import simplejson as json
5from decimal import Decimal as D, ROUND_DOWN 5from decimal import Decimal as D, ROUND_DOWN
6from datetime import date, datetime, timedelta 6import datetime
7import inspect 7import inspect
8from json import JSONDecodeError 8from json import JSONDecodeError
9from simplejson.errors import JSONDecodeError as SimpleJSONDecodeError 9from simplejson.errors import JSONDecodeError as SimpleJSONDecodeError
@@ -11,13 +11,16 @@ from simplejson.errors import JSONDecodeError as SimpleJSONDecodeError
11__all__ = ["Portfolio", "BalanceStore", "ReportStore", "TradeStore"] 11__all__ = ["Portfolio", "BalanceStore", "ReportStore", "TradeStore"]
12 12
13class ReportStore: 13class ReportStore:
14 def __init__(self, market, verbose_print=True): 14 def __init__(self, market, verbose_print=True, no_http_dup=False):
15 self.market = market 15 self.market = market
16 self.verbose_print = verbose_print 16 self.verbose_print = verbose_print
17 17
18 self.print_logs = [] 18 self.print_logs = []
19 self.logs = [] 19 self.logs = []
20 20
21 self.no_http_dup = no_http_dup
22 self.last_http = None
23
21 def merge(self, other_report): 24 def merge(self, other_report):
22 self.logs += other_report.logs 25 self.logs += other_report.logs
23 self.logs.sort(key=lambda x: x["date"]) 26 self.logs.sort(key=lambda x: x["date"])
@@ -26,19 +29,26 @@ class ReportStore:
26 self.print_logs.sort(key=lambda x: x[0]) 29 self.print_logs.sort(key=lambda x: x[0])
27 30
28 def print_log(self, message): 31 def print_log(self, message):
29 now = datetime.now() 32 now = datetime.datetime.now()
30 message = "{:%Y-%m-%d %H:%M:%S}: {}".format(now, str(message)) 33 message = "{:%Y-%m-%d %H:%M:%S}: {}".format(now, str(message))
31 self.print_logs.append([now, message]) 34 self.print_logs.append([now, message])
32 if self.verbose_print: 35 if self.verbose_print:
33 print(message) 36 print(message)
34 37
35 def add_log(self, hash_): 38 def add_log(self, hash_):
36 hash_["date"] = datetime.now() 39 hash_["date"] = datetime.datetime.now()
40 if self.market is not None:
41 hash_["user_id"] = self.market.user_id
42 hash_["market_id"] = self.market.market_id
43 else:
44 hash_["user_id"] = None
45 hash_["market_id"] = None
37 self.logs.append(hash_) 46 self.logs.append(hash_)
47 return hash_
38 48
39 @staticmethod 49 @staticmethod
40 def default_json_serial(obj): 50 def default_json_serial(obj):
41 if isinstance(obj, (datetime, date)): 51 if isinstance(obj, (datetime.datetime, datetime.date)):
42 return obj.isoformat() 52 return obj.isoformat()
43 return str(obj) 53 return str(obj)
44 54
@@ -188,7 +198,12 @@ class ReportStore:
188 "error": response.__class__.__name__, 198 "error": response.__class__.__name__,
189 "error_message": str(response), 199 "error_message": str(response),
190 }) 200 })
191 else: 201 self.last_http = None
202 elif self.no_http_dup and \
203 self.last_http is not None and \
204 self.last_http["url"] == url and \
205 self.last_http["method"] == method and \
206 self.last_http["response"] == response.text:
192 self.add_log({ 207 self.add_log({
193 "type": "http_request", 208 "type": "http_request",
194 "method": method, 209 "method": method,
@@ -196,7 +211,19 @@ class ReportStore:
196 "body": body, 211 "body": body,
197 "headers": headers, 212 "headers": headers,
198 "status": response.status_code, 213 "status": response.status_code,
199 "response": response.text 214 "response": None,
215 "response_same_as": self.last_http["date"]
216 })
217 else:
218 self.last_http = self.add_log({
219 "type": "http_request",
220 "method": method,
221 "url": url,
222 "body": body,
223 "headers": headers,
224 "status": response.status_code,
225 "response": response.text,
226 "response_same_as": None,
200 }) 227 })
201 228
202 def log_error(self, action, message=None, exception=None): 229 def log_error(self, action, message=None, exception=None):
@@ -222,13 +249,11 @@ class ReportStore:
222 "action": action, 249 "action": action,
223 }) 250 })
224 251
225 def log_market(self, args, user_id, market_id): 252 def log_market(self, args):
226 self.add_log({ 253 self.add_log({
227 "type": "market", 254 "type": "market",
228 "commit": "$Format:%H$", 255 "commit": "$Format:%H$",
229 "args": vars(args), 256 "args": vars(args),
230 "user_id": user_id,
231 "market_id": market_id,
232 }) 257 })
233 258
234class BalanceStore: 259class BalanceStore:
@@ -382,7 +407,7 @@ class Portfolio:
382 data = LockedVar(None) 407 data = LockedVar(None)
383 liquidities = LockedVar({}) 408 liquidities = LockedVar({})
384 last_date = LockedVar(None) 409 last_date = LockedVar(None)
385 report = LockedVar(ReportStore(None)) 410 report = LockedVar(ReportStore(None, no_http_dup=True))
386 worker = None 411 worker = None
387 worker_started = False 412 worker_started = False
388 worker_notify = None 413 worker_notify = None
@@ -418,11 +443,17 @@ class Portfolio:
418 raise RuntimeError("This method needs to be ran with the worker") 443 raise RuntimeError("This method needs to be ran with the worker")
419 while cls.worker_started: 444 while cls.worker_started:
420 cls.worker_notify.wait() 445 cls.worker_notify.wait()
421 cls.worker_notify.clear() 446 if cls.worker_started:
422 cls.report.print_log("Fetching cryptoportfolio") 447 cls.worker_notify.clear()
423 cls.get_cryptoportfolio(refetch=True) 448 cls.report.print_log("Fetching cryptoportfolio")
424 cls.callback.set() 449 cls.get_cryptoportfolio(refetch=True)
425 time.sleep(poll) 450 cls.callback.set()
451 time.sleep(poll)
452
453 @classmethod
454 def stop_worker(cls):
455 cls.worker_started = False
456 cls.worker_notify.set()
426 457
427 @classmethod 458 @classmethod
428 def notify_and_wait(cls): 459 def notify_and_wait(cls):
@@ -433,7 +464,7 @@ class Portfolio:
433 @classmethod 464 @classmethod
434 def wait_for_recent(cls, delta=4, poll=30): 465 def wait_for_recent(cls, delta=4, poll=30):
435 cls.get_cryptoportfolio() 466 cls.get_cryptoportfolio()
436 while cls.last_date.get() is None or datetime.now() - cls.last_date.get() > timedelta(delta): 467 while cls.last_date.get() is None or datetime.datetime.now() - cls.last_date.get() > datetime.timedelta(delta):
437 if cls.worker is None: 468 if cls.worker is None:
438 time.sleep(poll) 469 time.sleep(poll)
439 cls.report.print_log("Attempt to fetch up-to-date cryptoportfolio") 470 cls.report.print_log("Attempt to fetch up-to-date cryptoportfolio")
@@ -490,7 +521,7 @@ class Portfolio:
490 weights_hash = portfolio_hash["weights"] 521 weights_hash = portfolio_hash["weights"]
491 weights = {} 522 weights = {}
492 for i in range(len(weights_hash["_row"])): 523 for i in range(len(weights_hash["_row"])):
493 date = datetime.strptime(weights_hash["_row"][i], "%Y-%m-%d") 524 date = datetime.datetime.strptime(weights_hash["_row"][i], "%Y-%m-%d")
494 weights[date] = dict(filter( 525 weights[date] = dict(filter(
495 filter_weights, 526 filter_weights,
496 map(clean_weights(i), weights_hash.items()))) 527 map(clean_weights(i), weights_hash.items())))
@@ -504,8 +535,7 @@ class Portfolio:
504 "high": high_liquidity, 535 "high": high_liquidity,
505 }) 536 })
506 cls.last_date.set(max( 537 cls.last_date.set(max(
507 max(medium_liquidity.keys(), default=datetime(1, 1, 1)), 538 max(medium_liquidity.keys(), default=datetime.datetime(1, 1, 1)),
508 max(high_liquidity.keys(), default=datetime(1, 1, 1)) 539 max(high_liquidity.keys(), default=datetime.datetime(1, 1, 1))
509 )) 540 ))
510 541
511
diff --git a/tests/test_ccxt_wrapper.py b/tests/test_ccxt_wrapper.py
index d32469a..597fe5c 100644
--- a/tests/test_ccxt_wrapper.py
+++ b/tests/test_ccxt_wrapper.py
@@ -22,11 +22,13 @@ class poloniexETest(unittest.TestCase):
22 ccxt = market.ccxt.poloniexE() 22 ccxt = market.ccxt.poloniexE()
23 ccxt._market = mock.Mock 23 ccxt._market = mock.Mock
24 ccxt._market.report = mock.Mock() 24 ccxt._market.report = mock.Mock()
25 ccxt._market.market_id = 3
26 ccxt._market.user_id = 3
25 27
26 ccxt.session.request("GET", "URL", data="data", 28 ccxt.session.request("GET", "URL", data="data",
27 headers="headers") 29 headers={})
28 ccxt._market.report.log_http_request.assert_called_with('GET', 'URL', 'data', 30 ccxt._market.report.log_http_request.assert_called_with('GET', 'URL', 'data',
29 'headers', 'response') 31 {'X-market-id': '3', 'X-user-id': '3'}, 'response')
30 32
31 with self.subTest("Raising"),\ 33 with self.subTest("Raising"),\
32 mock.patch("market.ccxt.poloniexE.session") as session: 34 mock.patch("market.ccxt.poloniexE.session") as session:
@@ -35,12 +37,14 @@ class poloniexETest(unittest.TestCase):
35 ccxt = market.ccxt.poloniexE() 37 ccxt = market.ccxt.poloniexE()
36 ccxt._market = mock.Mock 38 ccxt._market = mock.Mock
37 ccxt._market.report = mock.Mock() 39 ccxt._market.report = mock.Mock()
40 ccxt._market.market_id = 3
41 ccxt._market.user_id = 3
38 42
39 with self.assertRaises(market.ccxt.RequestException, msg="Boo") as cm: 43 with self.assertRaises(market.ccxt.RequestException, msg="Boo") as cm:
40 ccxt.session.request("GET", "URL", data="data", 44 ccxt.session.request("GET", "URL", data="data",
41 headers="headers") 45 headers={})
42 ccxt._market.report.log_http_request.assert_called_with('GET', 'URL', 'data', 46 ccxt._market.report.log_http_request.assert_called_with('GET', 'URL', 'data',
43 'headers', cm.exception) 47 {'X-market-id': '3', 'X-user-id': '3'}, cm.exception)
44 48
45 49
46 def test_nanoseconds(self): 50 def test_nanoseconds(self):
diff --git a/tests/test_main.py b/tests/test_main.py
index 6396c07..e3a5677 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -179,7 +179,8 @@ class MainTest(WebMockTestCase):
179 mock.patch("main.parse_config") as parse_config,\ 179 mock.patch("main.parse_config") as parse_config,\
180 mock.patch("main.fetch_markets") as fetch_markets,\ 180 mock.patch("main.fetch_markets") as fetch_markets,\
181 mock.patch("main.process") as process,\ 181 mock.patch("main.process") as process,\
182 mock.patch("store.Portfolio.start_worker") as start: 182 mock.patch("store.Portfolio.start_worker") as start,\
183 mock.patch("store.Portfolio.stop_worker") as stop:
183 184
184 args_mock = mock.Mock() 185 args_mock = mock.Mock()
185 args_mock.parallel = True 186 args_mock.parallel = True
@@ -196,6 +197,7 @@ class MainTest(WebMockTestCase):
196 parse_config.assert_called_with(args_mock) 197 parse_config.assert_called_with(args_mock)
197 fetch_markets.assert_called_with("pg_config", "user") 198 fetch_markets.assert_called_with("pg_config", "user")
198 199
200 stop.assert_called_once_with()
199 start.assert_called_once_with() 201 start.assert_called_once_with()
200 self.assertEqual(2, process.call_count) 202 self.assertEqual(2, process.call_count)
201 process.assert_has_calls([ 203 process.assert_has_calls([
diff --git a/tests/test_market.py b/tests/test_market.py
index 82eeea8..14b23b5 100644
--- a/tests/test_market.py
+++ b/tests/test_market.py
@@ -548,7 +548,7 @@ class MarketTest(WebMockTestCase):
548 mock.patch.object(m, "report") as report,\ 548 mock.patch.object(m, "report") as report,\
549 mock.patch.object(m, "store_file_report") as file_report,\ 549 mock.patch.object(m, "store_file_report") as file_report,\
550 mock.patch.object(m, "store_database_report") as db_report,\ 550 mock.patch.object(m, "store_database_report") as db_report,\
551 mock.patch.object(market, "datetime") as time_mock: 551 mock.patch.object(market.datetime, "datetime") as time_mock:
552 552
553 time_mock.now.return_value = datetime.datetime(2018, 2, 25) 553 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
554 554
@@ -564,7 +564,7 @@ class MarketTest(WebMockTestCase):
564 mock.patch.object(m, "report") as report,\ 564 mock.patch.object(m, "report") as report,\
565 mock.patch.object(m, "store_file_report") as file_report,\ 565 mock.patch.object(m, "store_file_report") as file_report,\
566 mock.patch.object(m, "store_database_report") as db_report,\ 566 mock.patch.object(m, "store_database_report") as db_report,\
567 mock.patch.object(market, "datetime") as time_mock: 567 mock.patch.object(market.datetime, "datetime") as time_mock:
568 568
569 time_mock.now.return_value = datetime.datetime(2018, 2, 25) 569 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
570 570
@@ -580,7 +580,7 @@ class MarketTest(WebMockTestCase):
580 mock.patch.object(m, "report") as report,\ 580 mock.patch.object(m, "report") as report,\
581 mock.patch.object(m, "store_file_report") as file_report,\ 581 mock.patch.object(m, "store_file_report") as file_report,\
582 mock.patch.object(m, "store_database_report") as db_report,\ 582 mock.patch.object(m, "store_database_report") as db_report,\
583 mock.patch.object(market, "datetime") as time_mock: 583 mock.patch.object(market.datetime, "datetime") as time_mock:
584 584
585 time_mock.now.return_value = datetime.datetime(2018, 2, 25) 585 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
586 586
@@ -597,7 +597,7 @@ class MarketTest(WebMockTestCase):
597 mock.patch.object(m, "report") as report,\ 597 mock.patch.object(m, "report") as report,\
598 mock.patch.object(m, "store_file_report") as file_report,\ 598 mock.patch.object(m, "store_file_report") as file_report,\
599 mock.patch.object(m, "store_database_report") as db_report,\ 599 mock.patch.object(m, "store_database_report") as db_report,\
600 mock.patch.object(market, "datetime") as time_mock: 600 mock.patch.object(market.datetime, "datetime") as time_mock:
601 601
602 time_mock.now.return_value = datetime.datetime(2018, 2, 25) 602 time_mock.now.return_value = datetime.datetime(2018, 2, 25)
603 603
diff --git a/tests/test_portfolio.py b/tests/test_portfolio.py
index a1b95bf..98048ac 100644
--- a/tests/test_portfolio.py
+++ b/tests/test_portfolio.py
@@ -1742,7 +1742,7 @@ class MouvementTest(WebMockTestCase):
1742 self.assertEqual(42, mouvement.id) 1742 self.assertEqual(42, mouvement.id)
1743 self.assertEqual("buy", mouvement.action) 1743 self.assertEqual("buy", mouvement.action)
1744 self.assertEqual(D("0.0015"), mouvement.fee_rate) 1744 self.assertEqual(D("0.0015"), mouvement.fee_rate)
1745 self.assertEqual(portfolio.datetime(2017, 12, 30, 12, 0, 12), mouvement.date) 1745 self.assertEqual(portfolio.datetime.datetime(2017, 12, 30, 12, 0, 12), mouvement.date)
1746 self.assertEqual(D("0.1"), mouvement.rate) 1746 self.assertEqual(D("0.1"), mouvement.rate)
1747 self.assertEqual(portfolio.Amount("ETH", "10"), mouvement.total) 1747 self.assertEqual(portfolio.Amount("ETH", "10"), mouvement.total)
1748 self.assertEqual(portfolio.Amount("BTC", "1"), mouvement.total_in_base) 1748 self.assertEqual(portfolio.Amount("BTC", "1"), mouvement.total_in_base)
@@ -1780,7 +1780,7 @@ class MouvementTest(WebMockTestCase):
1780 as_json = mouvement.as_json() 1780 as_json = mouvement.as_json()
1781 1781
1782 self.assertEqual(D("0.0015"), as_json["fee_rate"]) 1782 self.assertEqual(D("0.0015"), as_json["fee_rate"])
1783 self.assertEqual(portfolio.datetime(2017, 12, 30, 12, 0, 12), as_json["date"]) 1783 self.assertEqual(portfolio.datetime.datetime(2017, 12, 30, 12, 0, 12), as_json["date"])
1784 self.assertEqual("buy", as_json["action"]) 1784 self.assertEqual("buy", as_json["action"])
1785 self.assertEqual(D("10"), as_json["total"]) 1785 self.assertEqual(D("10"), as_json["total"])
1786 self.assertEqual(D("1"), as_json["total_in_base"]) 1786 self.assertEqual(D("1"), as_json["total_in_base"])
diff --git a/tests/test_store.py b/tests/test_store.py
index c0b1fb9..2b51719 100644
--- a/tests/test_store.py
+++ b/tests/test_store.py
@@ -444,10 +444,20 @@ class BalanceStoreTest(WebMockTestCase):
444@unittest.skipUnless("unit" in limits, "Unit skipped") 444@unittest.skipUnless("unit" in limits, "Unit skipped")
445class ReportStoreTest(WebMockTestCase): 445class ReportStoreTest(WebMockTestCase):
446 def test_add_log(self): 446 def test_add_log(self):
447 report_store = market.ReportStore(self.m) 447 with self.subTest(market=self.m):
448 report_store.add_log({"foo": "bar"}) 448 self.m.user_id = 1
449 self.m.market_id = 3
450 report_store = market.ReportStore(self.m)
451 result = report_store.add_log({"foo": "bar"})
452
453 self.assertEqual({"foo": "bar", "date": mock.ANY, "user_id": 1, "market_id": 3}, result)
454 self.assertEqual(result, report_store.logs[0])
455
456 with self.subTest(market=None):
457 report_store = market.ReportStore(None)
458 result = report_store.add_log({"foo": "bar"})
449 459
450 self.assertEqual({"foo": "bar", "date": mock.ANY}, report_store.logs[0]) 460 self.assertEqual({"foo": "bar", "date": mock.ANY, "user_id": None, "market_id": None}, result)
451 461
452 def test_set_verbose(self): 462 def test_set_verbose(self):
453 report_store = market.ReportStore(self.m) 463 report_store = market.ReportStore(self.m)
@@ -460,6 +470,8 @@ class ReportStoreTest(WebMockTestCase):
460 self.assertFalse(report_store.verbose_print) 470 self.assertFalse(report_store.verbose_print)
461 471
462 def test_merge(self): 472 def test_merge(self):
473 self.m.user_id = 1
474 self.m.market_id = 3
463 report_store1 = market.ReportStore(self.m, verbose_print=False) 475 report_store1 = market.ReportStore(self.m, verbose_print=False)
464 report_store2 = market.ReportStore(None, verbose_print=False) 476 report_store2 = market.ReportStore(None, verbose_print=False)
465 477
@@ -478,7 +490,7 @@ class ReportStoreTest(WebMockTestCase):
478 with self.subTest(verbose=True),\ 490 with self.subTest(verbose=True),\
479 mock.patch.object(store, "datetime") as time_mock,\ 491 mock.patch.object(store, "datetime") as time_mock,\
480 mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock: 492 mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock:
481 time_mock.now.return_value = datetime.datetime(2018, 2, 25, 2, 20, 10) 493 time_mock.datetime.now.return_value = datetime.datetime(2018, 2, 25, 2, 20, 10)
482 report_store.set_verbose(True) 494 report_store.set_verbose(True)
483 report_store.print_log("Coucou") 495 report_store.print_log("Coucou")
484 report_store.print_log(portfolio.Amount("BTC", 1)) 496 report_store.print_log(portfolio.Amount("BTC", 1))
@@ -495,7 +507,7 @@ class ReportStoreTest(WebMockTestCase):
495 report_store = market.ReportStore(self.m) 507 report_store = market.ReportStore(self.m)
496 508
497 self.assertEqual("2018-02-24T00:00:00", 509 self.assertEqual("2018-02-24T00:00:00",
498 report_store.default_json_serial(portfolio.datetime(2018, 2, 24))) 510 report_store.default_json_serial(portfolio.datetime.datetime(2018, 2, 24)))
499 self.assertEqual("1.00000000 BTC", 511 self.assertEqual("1.00000000 BTC",
500 report_store.default_json_serial(portfolio.Amount("BTC", 1))) 512 report_store.default_json_serial(portfolio.Amount("BTC", 1)))
501 513
@@ -503,7 +515,7 @@ class ReportStoreTest(WebMockTestCase):
503 report_store = market.ReportStore(self.m) 515 report_store = market.ReportStore(self.m)
504 report_store.logs.append({"foo": "bar"}) 516 report_store.logs.append({"foo": "bar"})
505 self.assertEqual('[\n {\n "foo": "bar"\n }\n]', report_store.to_json()) 517 self.assertEqual('[\n {\n "foo": "bar"\n }\n]', report_store.to_json())
506 report_store.logs.append({"date": portfolio.datetime(2018, 2, 24)}) 518 report_store.logs.append({"date": portfolio.datetime.datetime(2018, 2, 24)})
507 self.assertEqual('[\n {\n "foo": "bar"\n },\n {\n "date": "2018-02-24T00:00:00"\n }\n]', report_store.to_json()) 519 self.assertEqual('[\n {\n "foo": "bar"\n },\n {\n "date": "2018-02-24T00:00:00"\n }\n]', report_store.to_json())
508 report_store.logs.append({"amount": portfolio.Amount("BTC", 1)}) 520 report_store.logs.append({"amount": portfolio.Amount("BTC", 1)})
509 self.assertEqual('[\n {\n "foo": "bar"\n },\n {\n "date": "2018-02-24T00:00:00"\n },\n {\n "amount": "1.00000000 BTC"\n }\n]', report_store.to_json()) 521 self.assertEqual('[\n {\n "foo": "bar"\n },\n {\n "date": "2018-02-24T00:00:00"\n },\n {\n "amount": "1.00000000 BTC"\n }\n]', report_store.to_json())
@@ -817,53 +829,99 @@ class ReportStoreTest(WebMockTestCase):
817 } 829 }
818 }) 830 })
819 831
820 @mock.patch.object(market.ReportStore, "print_log") 832 def test_log_http_request(self):
821 @mock.patch.object(market.ReportStore, "add_log") 833 with mock.patch.object(market.ReportStore, "add_log") as add_log:
822 def test_log_http_request(self, add_log, print_log): 834 report_store = market.ReportStore(self.m)
823 report_store = market.ReportStore(self.m) 835 response = mock.Mock()
824 response = mock.Mock() 836 response.status_code = 200
825 response.status_code = 200 837 response.text = "Hey"
826 response.text = "Hey"
827 838
828 report_store.log_http_request("method", "url", "body", 839 report_store.log_http_request("method", "url", "body",
829 "headers", response) 840 "headers", response)
830 print_log.assert_not_called() 841 add_log.assert_called_once_with({
831 add_log.assert_called_once_with({ 842 'type': 'http_request',
832 'type': 'http_request', 843 'method': 'method',
833 'method': 'method', 844 'url': 'url',
834 'url': 'url', 845 'body': 'body',
835 'body': 'body', 846 'headers': 'headers',
836 'headers': 'headers', 847 'status': 200,
837 'status': 200, 848 'response': 'Hey',
838 'response': 'Hey' 849 'response_same_as': None,
839 }) 850 })
840 851
841 add_log.reset_mock() 852 add_log.reset_mock()
842 report_store.log_http_request("method", "url", "body", 853 report_store.log_http_request("method", "url", "body",
843 "headers", ValueError("Foo")) 854 "headers", ValueError("Foo"))
844 add_log.assert_called_once_with({ 855 add_log.assert_called_once_with({
845 'type': 'http_request', 856 'type': 'http_request',
846 'method': 'method', 857 'method': 'method',
847 'url': 'url', 858 'url': 'url',
848 'body': 'body', 859 'body': 'body',
849 'headers': 'headers', 860 'headers': 'headers',
850 'status': -1, 861 'status': -1,
851 'response': None, 862 'response': None,
852 'error': 'ValueError', 863 'error': 'ValueError',
853 'error_message': 'Foo', 864 'error_message': 'Foo',
854 }) 865 })
866
867 with self.subTest(no_http_dup=True, duplicate=True):
868 self.m.user_id = 1
869 self.m.market_id = 3
870 report_store = market.ReportStore(self.m, no_http_dup=True)
871 original_add_log = report_store.add_log
872 with mock.patch.object(report_store, "add_log", side_effect=original_add_log) as add_log:
873 report_store.log_http_request("method", "url", "body",
874 "headers", response)
875 report_store.log_http_request("method", "url", "body",
876 "headers", response)
877 self.assertEqual(2, add_log.call_count)
878 self.assertIsNone(add_log.mock_calls[0][1][0]["response_same_as"])
879 self.assertIsNone(add_log.mock_calls[1][1][0]["response"])
880 self.assertEqual(add_log.mock_calls[0][1][0]["date"], add_log.mock_calls[1][1][0]["response_same_as"])
881 with self.subTest(no_http_dup=True, duplicate=False, case="Different call"):
882 self.m.user_id = 1
883 self.m.market_id = 3
884 report_store = market.ReportStore(self.m, no_http_dup=True)
885 original_add_log = report_store.add_log
886 with mock.patch.object(report_store, "add_log", side_effect=original_add_log) as add_log:
887 report_store.log_http_request("method", "url", "body",
888 "headers", response)
889 report_store.log_http_request("method2", "url", "body",
890 "headers", response)
891 self.assertEqual(2, add_log.call_count)
892 self.assertIsNone(add_log.mock_calls[0][1][0]["response_same_as"])
893 self.assertIsNone(add_log.mock_calls[1][1][0]["response_same_as"])
894 with self.subTest(no_http_dup=True, duplicate=False, case="Call inbetween"):
895 self.m.user_id = 1
896 self.m.market_id = 3
897 report_store = market.ReportStore(self.m, no_http_dup=True)
898 original_add_log = report_store.add_log
899
900 response2 = mock.Mock()
901 response2.status_code = 200
902 response2.text = "Hey there!"
903
904 with mock.patch.object(report_store, "add_log", side_effect=original_add_log) as add_log:
905 report_store.log_http_request("method", "url", "body",
906 "headers", response)
907 report_store.log_http_request("method", "url", "body",
908 "headers", response2)
909 report_store.log_http_request("method", "url", "body",
910 "headers", response)
911 self.assertEqual(3, add_log.call_count)
912 self.assertIsNone(add_log.mock_calls[0][1][0]["response_same_as"])
913 self.assertIsNone(add_log.mock_calls[1][1][0]["response_same_as"])
914 self.assertIsNone(add_log.mock_calls[2][1][0]["response_same_as"])
855 915
856 @mock.patch.object(market.ReportStore, "add_log") 916 @mock.patch.object(market.ReportStore, "add_log")
857 def test_log_market(self, add_log): 917 def test_log_market(self, add_log):
858 report_store = market.ReportStore(self.m) 918 report_store = market.ReportStore(self.m)
859 919
860 report_store.log_market(self.market_args(debug=True, quiet=False), 4, 1) 920 report_store.log_market(self.market_args(debug=True, quiet=False))
861 add_log.assert_called_once_with({ 921 add_log.assert_called_once_with({
862 "type": "market", 922 "type": "market",
863 "commit": "$Format:%H$", 923 "commit": "$Format:%H$",
864 "args": { "report_path": None, "debug": True, "quiet": False }, 924 "args": { "report_path": None, "debug": True, "quiet": False },
865 "user_id": 4,
866 "market_id": 1,
867 }) 925 })
868 926
869 @mock.patch.object(market.ReportStore, "print_log") 927 @mock.patch.object(market.ReportStore, "print_log")
@@ -1034,7 +1092,7 @@ class PortfolioTest(WebMockTestCase):
1034 'SC': (D("0.0623"), "long"), 1092 'SC': (D("0.0623"), "long"),
1035 'ZEC': (D("0.3701"), "long"), 1093 'ZEC': (D("0.3701"), "long"),
1036 } 1094 }
1037 date = portfolio.datetime(2018, 1, 8) 1095 date = portfolio.datetime.datetime(2018, 1, 8)
1038 self.assertDictEqual(expected, liquidities["high"][date]) 1096 self.assertDictEqual(expected, liquidities["high"][date])
1039 1097
1040 expected = { 1098 expected = {
@@ -1051,7 +1109,7 @@ class PortfolioTest(WebMockTestCase):
1051 'XCP': (D("0.1"), "long"), 1109 'XCP': (D("0.1"), "long"),
1052 } 1110 }
1053 self.assertDictEqual(expected, liquidities["medium"][date]) 1111 self.assertDictEqual(expected, liquidities["medium"][date])
1054 self.assertEqual(portfolio.datetime(2018, 1, 15), market.Portfolio.last_date.get()) 1112 self.assertEqual(portfolio.datetime.datetime(2018, 1, 15), market.Portfolio.last_date.get())
1055 1113
1056 with self.subTest(description="Missing weight"): 1114 with self.subTest(description="Missing weight"):
1057 data = store.json.loads(self.json_response, parse_int=D, parse_float=D) 1115 data = store.json.loads(self.json_response, parse_int=D, parse_float=D)
@@ -1105,9 +1163,9 @@ class PortfolioTest(WebMockTestCase):
1105 else: 1163 else:
1106 self.assertFalse(refetch) 1164 self.assertFalse(refetch)
1107 self.call_count += 1 1165 self.call_count += 1
1108 market.Portfolio.last_date = store.LockedVar(store.datetime.now()\ 1166 market.Portfolio.last_date = store.LockedVar(store.datetime.datetime.now()\
1109 - store.timedelta(10)\ 1167 - store.datetime.timedelta(10)\
1110 + store.timedelta(self.call_count)) 1168 + store.datetime.timedelta(self.call_count))
1111 get_cryptoportfolio.side_effect = _get 1169 get_cryptoportfolio.side_effect = _get
1112 1170
1113 market.Portfolio.wait_for_recent() 1171 market.Portfolio.wait_for_recent()
@@ -1166,6 +1224,19 @@ class PortfolioTest(WebMockTestCase):
1166 self.assertTrue(store.Portfolio.worker_started) 1224 self.assertTrue(store.Portfolio.worker_started)
1167 1225
1168 self.assertFalse(store.Portfolio.worker.is_alive()) 1226 self.assertFalse(store.Portfolio.worker.is_alive())
1227 self.assertEqual(1, threading.active_count())
1228
1229 def test_stop_worker(self):
1230 with mock.patch.object(store.Portfolio, "get_cryptoportfolio") as get,\
1231 mock.patch.object(store.Portfolio, "report") as report,\
1232 mock.patch.object(store.time, "sleep") as sleep:
1233 store.Portfolio.start_worker(poll=3)
1234 store.Portfolio.stop_worker()
1235 store.Portfolio.worker.join()
1236 get.assert_not_called()
1237 report.assert_not_called()
1238 sleep.assert_not_called()
1239 self.assertFalse(store.Portfolio.worker.is_alive())
1169 1240
1170 def test_wait_for_notification(self): 1241 def test_wait_for_notification(self):
1171 with self.assertRaises(RuntimeError): 1242 with self.assertRaises(RuntimeError):
@@ -1189,7 +1260,7 @@ class PortfolioTest(WebMockTestCase):
1189 store.Portfolio.callback.clear() 1260 store.Portfolio.callback.clear()
1190 store.Portfolio.worker_started = False 1261 store.Portfolio.worker_started = False
1191 store.Portfolio.worker_notify.set() 1262 store.Portfolio.worker_notify.set()
1192 store.Portfolio.callback.wait() 1263 store.Portfolio.worker.join()
1193 self.assertFalse(store.Portfolio.worker.is_alive()) 1264 self.assertFalse(store.Portfolio.worker.is_alive())
1194 1265
1195 def test_notify_and_wait(self): 1266 def test_notify_and_wait(self):