From c7c1e0b26821fdd5622f81fb456f1028d4c9ab09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Sat, 24 Mar 2018 15:18:31 +0100 Subject: Add retry facility for api call timeouts Fixes https://git.immae.eu/mantisbt/view.php?id=40 --- ccxt_wrapper.py | 30 ++++++++++++++++++++++++++++++ test.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/ccxt_wrapper.py b/ccxt_wrapper.py index d37c306..c500659 100644 --- a/ccxt_wrapper.py +++ b/ccxt_wrapper.py @@ -1,12 +1,42 @@ from ccxt import * import decimal import time +from retry.api import retry_call +import re def _cw_exchange_sum(self, *args): return sum([arg for arg in args if isinstance(arg, (float, int, decimal.Decimal))]) Exchange.sum = _cw_exchange_sum class poloniexE(poloniex): + RETRIABLE_CALLS = [ + re.compile(r"^return"), + re.compile(r"^cancel"), + re.compile(r"^closeMarginPosition$"), + re.compile(r"^getMarginPosition$"), + ] + + def request(self, path, api='public', method='GET', params={}, headers=None, body=None): + """ + Wrapped to allow retry of non-posting requests" + """ + + origin_request = super(poloniexE, self).request + kwargs = { + "api": api, + "method": method, + "params": params, + "headers": headers, + "body": body + } + + retriable = any(re.match(call, path) for call in self.RETRIABLE_CALLS) + if api == "public" or method == "GET" or retriable: + return retry_call(origin_request, fargs=[path], fkwargs=kwargs, + tries=10, delay=1, exceptions=(RequestTimeout,)) + else: + return origin_request(path, **kwargs) + @staticmethod def nanoseconds(): return int(time.time() * 1000000000) diff --git a/test.py b/test.py index 637a305..40c64a9 100644 --- a/test.py +++ b/test.py @@ -80,6 +80,58 @@ class poloniexETest(unittest.TestCase): time.return_value = 123456.7890123456 self.assertEqual(123456789012345, self.s.nonce()) + def test_request(self): + with mock.patch.object(market.ccxt.poloniex, "request") as request,\ + mock.patch("market.ccxt.retry_call") as retry_call: + with self.subTest(wrapped=True): + with self.subTest(desc="public"): + self.s.request("foo") + retry_call.assert_called_with(request, + delay=1, tries=10, fargs=["foo"], + fkwargs={'api': 'public', 'method': 'GET', 'params': {}, 'headers': None, 'body': None}, + exceptions=(market.ccxt.RequestTimeout,)) + request.assert_not_called() + + with self.subTest(desc="private GET"): + self.s.request("foo", api="private") + retry_call.assert_called_with(request, + delay=1, tries=10, fargs=["foo"], + fkwargs={'api': 'private', 'method': 'GET', 'params': {}, 'headers': None, 'body': None}, + exceptions=(market.ccxt.RequestTimeout,)) + request.assert_not_called() + + with self.subTest(desc="private POST regexp"): + self.s.request("returnFoo", api="private", method="POST") + retry_call.assert_called_with(request, + delay=1, tries=10, fargs=["returnFoo"], + fkwargs={'api': 'private', 'method': 'POST', 'params': {}, 'headers': None, 'body': None}, + exceptions=(market.ccxt.RequestTimeout,)) + request.assert_not_called() + + with self.subTest(desc="private POST non-regexp"): + self.s.request("getMarginPosition", api="private", method="POST") + retry_call.assert_called_with(request, + delay=1, tries=10, fargs=["getMarginPosition"], + fkwargs={'api': 'private', 'method': 'POST', 'params': {}, 'headers': None, 'body': None}, + exceptions=(market.ccxt.RequestTimeout,)) + request.assert_not_called() + retry_call.reset_mock() + request.reset_mock() + with self.subTest(wrapped=False): + with self.subTest(desc="private POST non-matching regexp"): + self.s.request("marginBuy", api="private", method="POST") + request.assert_called_with("marginBuy", + api="private", method="POST", params={}, + headers=None, body=None) + retry_call.assert_not_called() + + with self.subTest(desc="private POST non-matching non-regexp"): + self.s.request("closeMarginPositionOther", api="private", method="POST") + request.assert_called_with("closeMarginPositionOther", + api="private", method="POST", params={}, + headers=None, body=None) + retry_call.assert_not_called() + def test_order_precision(self): self.assertEqual(8, self.s.order_precision("FOO")) -- cgit v1.2.3 From 445b4a7712fb7fe45e17b6b76356dd3be42dd900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Sat, 24 Mar 2018 15:18:56 +0100 Subject: Move request wrapper to ccxt --- ccxt_wrapper.py | 15 +++++++++++++++ market.py | 12 ------------ test.py | 18 ++++++++++++------ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/ccxt_wrapper.py b/ccxt_wrapper.py index c500659..4ed37d9 100644 --- a/ccxt_wrapper.py +++ b/ccxt_wrapper.py @@ -37,6 +37,21 @@ class poloniexE(poloniex): else: return origin_request(path, **kwargs) + def __init__(self, *args, **kwargs): + super(poloniexE, self).__init__(*args, **kwargs) + + # For requests logging + self.session.origin_request = self.session.request + self.session._parent = self + + def request_wrap(self, *args, **kwargs): + r = self.origin_request(*args, **kwargs) + self._parent._market.report.log_http_request(args[0], + args[1], kwargs["data"], kwargs["headers"], r) + return r + self.session.request = request_wrap.__get__(self.session, + self.session.__class__) + @staticmethod def nanoseconds(): return int(time.time() * 1000000000) diff --git a/market.py b/market.py index 496ec45..055967c 100644 --- a/market.py +++ b/market.py @@ -33,18 +33,6 @@ class Market: ccxt_instance = ccxt.poloniexE(config) - # For requests logging - ccxt_instance.session.origin_request = ccxt_instance.session.request - ccxt_instance.session._parent = ccxt_instance - - def request_wrap(self, *args, **kwargs): - r = self.origin_request(*args, **kwargs) - self._parent._market.report.log_http_request(args[0], - args[1], kwargs["data"], kwargs["headers"], r) - return r - ccxt_instance.session.request = request_wrap.__get__(ccxt_instance.session, - ccxt_instance.session.__class__) - return cls(ccxt_instance, args, **kwargs) def store_report(self): diff --git a/test.py b/test.py index 40c64a9..18616c1 100644 --- a/test.py +++ b/test.py @@ -70,6 +70,18 @@ class poloniexETest(unittest.TestCase): self.wm.stop() super(poloniexETest, self).tearDown() + def test__init(self): + with mock.patch("market.ccxt.poloniexE.session") as session: + session.request.return_value = "response" + ccxt = market.ccxt.poloniexE() + ccxt._market = mock.Mock + ccxt._market.report = mock.Mock() + + ccxt.session.request("GET", "URL", data="data", + headers="headers") + ccxt._market.report.log_http_request.assert_called_with('GET', 'URL', 'data', + 'headers', 'response') + def test_nanoseconds(self): with mock.patch.object(market.ccxt.time, "time") as time: time.return_value = 123456.7890123456 @@ -1177,17 +1189,11 @@ class MarketTest(WebMockTestCase): def test_from_config(self, ccxt): with mock.patch("market.ReportStore"): ccxt.poloniexE.return_value = self.ccxt - self.ccxt.session.request.return_value = "response" m = market.Market.from_config({"key": "key", "secred": "secret"}, self.market_args()) self.assertEqual(self.ccxt, m.ccxt) - self.ccxt.session.request("GET", "URL", data="data", - headers="headers") - m.report.log_http_request.assert_called_with('GET', 'URL', 'data', - 'headers', 'response') - m = market.Market.from_config({"key": "key", "secred": "secret"}, self.market_args(debug=True)) self.assertEqual(True, m.debug) -- cgit v1.2.3