X-Git-Url: https://git.immae.eu/?p=perso%2FImmae%2FProjets%2FCryptomonnaies%2FCryptoportfolio%2FTrader.git;a=blobdiff_plain;f=portfolio.py;h=1d291069bfa08c7cb497e90046de010e6d7f2e02;hp=43a39c4506c7caa6ba6fdbea4f48c5c6c10e4397;hb=HEAD;hpb=f86ee14037646bedc3a3dee4a48f085308981757

diff --git a/portfolio.py b/portfolio.py
index 43a39c4..1d29106 100644
--- a/portfolio.py
+++ b/portfolio.py
@@ -1,87 +1,13 @@
-import time
-from datetime import datetime, timedelta
+import datetime
+from retry import retry
 from decimal import Decimal as D, ROUND_DOWN
-from json import JSONDecodeError
-from simplejson.errors import JSONDecodeError as SimpleJSONDecodeError
-from ccxt import ExchangeError, ExchangeNotAvailable
-import requests
-
-# FIXME: correctly handle web call timeouts
-
-class Portfolio:
-    URL = "https://cryptoportfolio.io/wp-content/uploads/portfolio/json/cryptoportfolio.json"
-    liquidities = {}
-    data = None
-    last_date = None
-
-    @classmethod
-    def wait_for_recent(cls, market, delta=4):
-        cls.repartition(market, refetch=True)
-        while cls.last_date is None or datetime.now() - cls.last_date > timedelta(delta):
-            time.sleep(30)
-            market.report.print_log("Attempt to fetch up-to-date cryptoportfolio")
-            cls.repartition(market, refetch=True)
-
-    @classmethod
-    def repartition(cls, market, liquidity="medium", refetch=False):
-        cls.parse_cryptoportfolio(market, refetch=refetch)
-        liquidities = cls.liquidities[liquidity]
-        return liquidities[cls.last_date]
-
-    @classmethod
-    def get_cryptoportfolio(cls, market):
-        try:
-            r = requests.get(cls.URL)
-            market.report.log_http_request(r.request.method,
-                    r.request.url, r.request.body, r.request.headers, r)
-        except Exception as e:
-            market.report.log_error("get_cryptoportfolio", exception=e)
-            return
-        try:
-            cls.data = r.json(parse_int=D, parse_float=D)
-        except (JSONDecodeError, SimpleJSONDecodeError):
-            cls.data = None
-
-    @classmethod
-    def parse_cryptoportfolio(cls, market, refetch=False):
-        if refetch or cls.data is None:
-            cls.get_cryptoportfolio(market)
-
-        def filter_weights(weight_hash):
-            if weight_hash[1][0] == 0:
-                return False
-            if weight_hash[0] == "_row":
-                return False
-            return True
-
-        def clean_weights(i):
-            def clean_weights_(h):
-                if h[0].endswith("s"):
-                    return [h[0][0:-1], (h[1][i], "short")]
-                else:
-                    return [h[0], (h[1][i], "long")]
-            return clean_weights_
-
-        def parse_weights(portfolio_hash):
-            weights_hash = portfolio_hash["weights"]
-            weights = {}
-            for i in range(len(weights_hash["_row"])):
-                date = datetime.strptime(weights_hash["_row"][i], "%Y-%m-%d")
-                weights[date] = dict(filter(
-                        filter_weights,
-                        map(clean_weights(i), weights_hash.items())))
-            return weights
-
-        high_liquidity = parse_weights(cls.data["portfolio_1"])
-        medium_liquidity = parse_weights(cls.data["portfolio_2"])
-
-        cls.liquidities = {
-                "medium": medium_liquidity,
-                "high":   high_liquidity,
-                }
-        cls.last_date = max(max(medium_liquidity.keys()), max(high_liquidity.keys()))
+from ccxt import ExchangeError, InsufficientFunds, ExchangeNotAvailable, InvalidOrder, OrderNotCached, OrderNotFound, RequestTimeout, InvalidNonce
 
 class Computation:
+    @staticmethod
+    def eat_several(market):
+        return lambda x, y: market.ccxt.fetch_nth_order_book(x["symbol"], y, 15)
+
     computations = {
             "default": lambda x, y: x[y],
             "average": lambda x, y: x["average"],
@@ -126,7 +52,7 @@ class Amount:
                     ticker=asset_ticker,
                     rate=rate)
         else:
-            raise Exception("This asset is not available in the chosen market")
+            return Amount(other_currency, 0, linked_to=self, ticker=None, rate=0)
 
     def as_json(self):
         return {
@@ -225,8 +151,8 @@ class Amount:
 
 class Balance:
     base_keys = ["total", "exchange_total", "exchange_used",
-            "exchange_free", "margin_total", "margin_borrowed",
-            "margin_free"]
+            "exchange_free", "margin_total", "margin_in_position",
+            "margin_available", "margin_borrowed", "margin_pending_gain"]
 
     def __init__(self, currency, hash_):
         self.currency = currency
@@ -239,8 +165,8 @@ class Balance:
             base_currency = hash_["margin_borrowed_base_currency"]
             for key in [
                     "margin_liquidation_price",
-                    "margin_pending_gain",
                     "margin_lending_fees",
+                    "margin_pending_base_gain",
                     "margin_borrowed_base_price"
                     ]:
                 setattr(self, key, Amount(base_currency, hash_.get(key, 0)))
@@ -260,12 +186,12 @@ class Balance:
             exchange = ""
 
         if self.margin_total > 0:
-            if self.margin_free != 0 and self.margin_borrowed != 0:
-                margin = " Margin: [✔{} + borrowed {} = {}]".format(str(self.margin_free), str(self.margin_borrowed), str(self.margin_total))
-            elif self.margin_free != 0:
-                margin = " Margin: [✔{}]".format(str(self.margin_free))
+            if self.margin_available != 0 and self.margin_in_position != 0:
+                margin = " Margin: [✔{} + ❌{} = {}]".format(str(self.margin_available), str(self.margin_in_position), str(self.margin_total))
+            elif self.margin_available != 0:
+                margin = " Margin: [✔{}]".format(str(self.margin_available))
             else:
-                margin = " Margin: [borrowed {}]".format(str(self.margin_borrowed))
+                margin = " Margin: [❌{}]".format(str(self.margin_in_position))
         elif self.margin_total < 0:
             margin = " Margin: [{} @@ {}/{}]".format(str(self.margin_total),
                     str(self.margin_borrowed_base_price),
@@ -289,6 +215,9 @@ class Trade:
         self.value_to = value_to
         self.orders = []
         self.market = market
+        self.closed = False
+        self.inverted = None
+        assert self.value_from.value * self.value_to.value >= 0
         assert self.value_from.currency == self.value_to.currency
         if self.value_from != 0:
             assert self.value_from.linked_to is not None and self.value_from.linked_to.currency == self.currency
@@ -296,6 +225,10 @@ class Trade:
             self.value_from.linked_to = Amount(self.currency, 0)
         self.base_currency = self.value_from.currency
 
+    @property
+    def delta(self):
+        return self.value_to - self.value_from
+
     @property
     def action(self):
         if self.value_from == self.value_to:
@@ -308,8 +241,8 @@ class Trade:
         else:
             return "dispose"
 
-    def order_action(self, inverted):
-        if (self.value_from < self.value_to) != inverted:
+    def order_action(self):
+        if (self.value_from < self.value_to) != self.inverted:
             return "buy"
         else:
             return "sell"
@@ -321,60 +254,84 @@ class Trade:
         else:
             return "long"
 
-    def filled_amount(self, in_base_currency=False):
+    @property
+    def pending(self):
+        return not (self.is_fullfiled or self.closed)
+
+    def close(self):
+        for order in self.orders:
+            order.cancel()
+        self.closed = True
+
+    @property
+    def is_fullfiled(self):
+        return abs(self.filled_amount(in_base_currency=(not self.inverted), refetch=True)) >= abs(self.delta)
+
+    def filled_amount(self, in_base_currency=False, refetch=False):
         filled_amount = 0
         for order in self.orders:
-            filled_amount += order.filled_amount(in_base_currency=in_base_currency)
+            filled_amount += order.filled_amount(in_base_currency=in_base_currency, refetch=refetch)
         return filled_amount
 
+    tick_actions = {
+            0: ["waiting", None],
+            1: ["waiting", None],
+            2: ["adjusting", lambda x, y: (x[y] + x["average"]) / 2],
+            3: ["waiting", None],
+            4: ["waiting", None],
+            5: ["adjusting", lambda x, y: (x[y]*2 + x["average"]) / 3],
+            6: ["waiting", None],
+            7: ["market_fallback", "default"],
+            }
+
+    def tick_actions_recreate(self, tick, default="average"):
+        return ([default] + \
+                [ y[1] for x, y in self.tick_actions.items() if x <= tick and y[1] is not None ])[-1]
+
     def update_order(self, order, tick):
-        new_order = None
-        if tick in [0, 1, 3, 4, 6]:
-            update = "waiting"
-            compute_value = None
-        elif tick == 2:
-            update = "adjusting"
-            compute_value = 'lambda x, y: (x[y] + x["average"]) / 2'
-            new_order = self.prepare_order(compute_value=lambda x, y: (x[y] + x["average"]) / 2)
-        elif tick ==5:
-            update = "adjusting"
-            compute_value = 'lambda x, y: (x[y]*2 + x["average"]) / 3'
-            new_order = self.prepare_order(compute_value=lambda x, y: (x[y]*2 + x["average"]) / 3)
-        elif tick >= 7:
-            if (tick - 7) % 3 == 0:
-                new_order = self.prepare_order(compute_value="default")
+        if tick in self.tick_actions:
+            update, compute_value = self.tick_actions[tick]
+        elif tick % 3 == 1:
+            if tick < 20:
                 update = "market_adjust"
                 compute_value = "default"
             else:
-                update = "waiting"
-                compute_value = None
-            if tick == 7:
-                update = "market_fallback"
+                update = "market_adjust_eat"
+                compute_value = Computation.eat_several(self.market)
+        else:
+            update = "waiting"
+            compute_value = None
+
+        if compute_value is not None:
+            order.cancel()
+            new_order = self.prepare_order(compute_value=compute_value)
+        else:
+            new_order = None
 
         self.market.report.log_order(order, tick, update=update,
                 compute_value=compute_value, new_order=new_order)
 
         if new_order is not None:
-            order.cancel()
             new_order.run()
             self.market.report.log_order(order, tick, new_order=new_order)
 
-    def prepare_order(self, compute_value="default"):
+    def prepare_order(self, close_if_possible=None, compute_value="default"):
         if self.action is None:
             return None
         ticker = self.market.get_ticker(self.currency, self.base_currency)
-        inverted = ticker["inverted"]
-        if inverted:
+        if ticker is None:
+            self.market.report.log_error("prepare_order",
+                    message="Unknown ticker {}/{}".format(self.currency, self.base_currency))
+            return None
+        self.inverted = ticker["inverted"]
+        if self.inverted:
             ticker = ticker["original"]
-        rate = Computation.compute_value(ticker, self.order_action(inverted), compute_value=compute_value)
+        rate = Computation.compute_value(ticker, self.order_action(), compute_value=compute_value)
 
-        #TODO: store when the order is considered filled
-        # FIXME: Dust amount should be removed from there if they werent
-        # honored in other sales
-        delta_in_base = abs(self.value_from - self.value_to)
+        delta_in_base = abs(self.delta)
         # 9 BTC's worth of move (10 - 1 or 1 - 10 depending on case)
 
-        if not inverted:
+        if not self.inverted:
             base_currency = self.base_currency
             # BTC
             if self.action == "dispose":
@@ -425,13 +382,14 @@ class Trade:
                 delta = delta - filled
                 # I already sold 4 BTC, only 5 left
 
-        close_if_possible = (self.value_to == 0)
+        if close_if_possible is None:
+            close_if_possible = (self.value_to == 0)
 
         if delta <= 0:
             self.market.report.log_error("prepare_order", message="Less to do than already filled: {}".format(delta))
             return None
 
-        order = Order(self.order_action(inverted),
+        order = Order(self.order_action(),
             delta, rate, base_currency, self.trade_type,
             self.market, self, close_if_possible=close_if_possible)
         self.orders.append(order)
@@ -447,11 +405,19 @@ class Trade:
                 }
 
     def __repr__(self):
-        return "Trade({} -> {} in {}, {})".format(
+        if self.closed and not self.is_fullfiled:
+            closed = " ❌"
+        elif self.is_fullfiled:
+            closed = " ✔"
+        else:
+            closed = ""
+
+        return "Trade({} -> {} in {}, {}{})".format(
                 self.value_from,
                 self.value_to,
                 self.currency,
-                self.action)
+                self.action,
+                closed)
 
     def print_with_order(self, ind=""):
         self.market.report.print_log("{}{}".format(ind, self))
@@ -460,6 +426,9 @@ class Trade:
             for mouvement in order.mouvements:
                 self.market.report.print_log("{}\t\t{}".format(ind, mouvement))
 
+class RetryException(Exception):
+    pass
+
 class Order:
     def __init__(self, action, amount, rate, base_currency, trade_type, market,
             trade, close_if_possible=False):
@@ -475,7 +444,8 @@ class Order:
         self.trade = trade
         self.close_if_possible = close_if_possible
         self.id = None
-        self.fetch_cache_timestamp = None
+        self.tries = 0
+        self.start_date = None
 
     def as_json(self):
         return {
@@ -519,27 +489,59 @@ class Order:
 
     @property
     def finished(self):
-        return self.status == "closed" or self.status == "canceled" or self.status == "error"
+        return self.status.startswith("closed") or self.status == "canceled" or self.status == "error"
 
+    @retry((InsufficientFunds, RetryException, InvalidNonce))
     def run(self):
+        self.tries += 1
         symbol = "{}/{}".format(self.amount.currency, self.base_currency)
         amount = round(self.amount, self.market.ccxt.order_precision(symbol)).value
 
+        action = "market.ccxt.create_order('{}', 'limit', '{}', {}, price={}, account={})".format(symbol, self.action, amount, self.rate, self.account)
         if self.market.debug:
-            self.market.report.log_debug_action("market.ccxt.create_order('{}', 'limit', '{}', {}, price={}, account={})".format(
-                symbol, self.action, amount, self.rate, self.account))
+            self.market.report.log_debug_action(action)
             self.results.append({"debug": True, "id": -1})
         else:
+            self.start_date = datetime.datetime.now()
             try:
                 self.results.append(self.market.ccxt.create_order(symbol, 'limit', self.action, amount, price=self.rate, account=self.account))
-            except ExchangeNotAvailable:
+            except InvalidOrder:
                 # Impossible to honor the order (dust amount)
                 self.status = "closed"
                 self.mark_finished_order()
                 return
+            except InvalidNonce as e:
+                if self.tries < 5:
+                    self.market.report.log_error(action, message="Retrying after invalid nonce", exception=e)
+                    raise e
+                else:
+                    self.market.report.log_error(action, message="Giving up {} after invalid nonce".format(self), exception=e)
+                    self.status = "error"
+                    return
+            except RequestTimeout as e:
+                if not self.retrieve_order():
+                    if self.tries < 5:
+                        self.market.report.log_error(action, message="Retrying after timeout", exception=e)
+                        # We make a specific call in case retrieve_order
+                        # would raise itself
+                        raise RetryException
+                    else:
+                        self.market.report.log_error(action, message="Giving up {} after timeouts".format(self), exception=e)
+                        self.status = "error"
+                        return
+                else:
+                    self.market.report.log_error(action, message="Timeout, found the order")
+            except InsufficientFunds as e:
+                if self.tries < 5:
+                    self.market.report.log_error(action, message="Retrying with reduced amount", exception=e)
+                    self.amount = self.amount * D("0.99")
+                    raise e
+                else:
+                    self.market.report.log_error(action, message="Giving up {}".format(self), exception=e)
+                    self.status = "error"
+                    return
             except Exception as e:
                 self.status = "error"
-                action = "market.ccxt.create_order('{}', 'limit', '{}', {}, price={}, account={})".format(symbol, self.action, amount, self.rate, self.account)
                 self.market.report.log_error(action, exception=e)
                 return
         self.id = self.results[0]["id"]
@@ -552,47 +554,50 @@ class Order:
         # other states are "closed" and "canceled"
         if not self.finished:
             self.fetch()
-            if self.finished:
-                self.mark_finished_order()
         return self.status
 
+    def mark_disappeared_order(self):
+        if self.status.startswith("closed") and \
+                len(self.mouvements) > 0 and \
+                self.mouvements[-1].total_in_base == 0:
+            self.status = "error_disappeared"
+
     def mark_finished_order(self):
-        if self.market.debug:
+        if self.status.startswith("closed") and self.market.debug:
             self.market.report.log_debug_action("Mark {} as finished".format(self))
             return
-        if self.status == "closed":
+        if self.status.startswith("closed"):
             if self.trade_type == "short" and self.action == "buy" and self.close_if_possible:
                 self.market.ccxt.close_margin_position(self.amount.currency, self.base_currency)
 
-    def fetch(self, force=False):
+    def fetch(self):
         if self.market.debug:
             self.market.report.log_debug_action("Fetching {}".format(self))
             return
-        if (not force and self.fetch_cache_timestamp is not None
-                and time.time() - self.fetch_cache_timestamp < 10):
-            return
-        self.fetch_cache_timestamp = time.time()
-
-        result = self.market.ccxt.fetch_order(self.id)
-        self.results.append(result)
+        try:
+            result = self.market.ccxt.fetch_order(self.id)
+            self.results.append(result)
+            self.status = result["status"]
+            # Time at which the order started
+            self.timestamp = result["datetime"]
+        except OrderNotCached:
+            self.status = "closed_unknown"
 
-        self.status = result["status"]
-        # Time at which the order started
-        self.timestamp = result["datetime"]
         self.fetch_mouvements()
 
-        # FIXME: consider open order with dust remaining as closed
+        self.mark_disappeared_order()
+        self.mark_dust_amount_remaining_order()
+        self.mark_finished_order()
 
-    def dust_amount_remaining(self):
-        return self.remaining_amount() < Amount(self.amount.currency, D("0.001"))
+    def mark_dust_amount_remaining_order(self):
+        if self.status == "open" and self.market.ccxt.is_dust_trade(self.remaining_amount().value, self.rate):
+            self.status = "closed_dust_remaining"
 
-    def remaining_amount(self):
-        if self.status == "open":
-            self.fetch()
-        return self.amount - self.filled_amount()
+    def remaining_amount(self, refetch=False):
+        return self.amount - self.filled_amount(refetch=refetch)
 
-    def filled_amount(self, in_base_currency=False):
-        if self.status == "open":
+    def filled_amount(self, in_base_currency=False, refetch=False):
+        if refetch and self.status == "open":
             self.fetch()
         filled_amount = 0
         for mouvement in self.mouvements:
@@ -612,14 +617,66 @@ class Order:
         for mouvement_hash in mouvements:
             self.mouvements.append(Mouvement(self.amount.currency,
                 self.base_currency, mouvement_hash))
+        self.mouvements.sort(key= lambda x: x.date)
 
     def cancel(self):
         if self.market.debug:
             self.market.report.log_debug_action("Mark {} as cancelled".format(self))
             self.status = "canceled"
             return
-        self.market.ccxt.cancel_order(self.id)
-        self.fetch()
+        if (self.status == "closed_dust_remaining" or self.open) and self.id is not None:
+            try:
+                self.market.ccxt.cancel_order(self.id)
+            except OrderNotFound as e: # Closed inbetween
+                self.market.report.log_error("cancel_order", message="Already cancelled order", exception=e)
+            self.fetch()
+
+    def retrieve_order(self):
+        symbol = "{}/{}".format(self.amount.currency, self.base_currency)
+        amount = round(self.amount, self.market.ccxt.order_precision(symbol)).value
+        start_timestamp = self.start_date.timestamp() - 5
+
+        similar_open_orders = self.market.ccxt.fetch_orders(symbol=symbol, since=start_timestamp)
+        for order in similar_open_orders:
+            if (order["info"]["margin"] == 1 and self.account == "exchange") or\
+                    (order["info"]["margin"] != 1 and self.account == "margin"):
+                i_m_tested = True # coverage bug ?!
+                continue
+            if order["info"]["side"] != self.action:
+                continue
+            amount_diff = round(
+                    abs(D(order["info"]["startingAmount"]) - amount),
+                    self.market.ccxt.order_precision(symbol))
+            rate_diff = round(
+                    abs(D(order["info"]["rate"]) - self.rate),
+                    self.market.ccxt.order_precision(symbol))
+            if amount_diff != 0 or rate_diff != 0:
+                continue
+            self.results.append({"id": order["id"]})
+            return True
+
+        similar_trades = self.market.ccxt.fetch_my_trades(symbol=symbol, since=start_timestamp)
+        for order_id in sorted(list(map(lambda x: x["order"], similar_trades))):
+            trades = list(filter(lambda x: x["order"] == order_id, similar_trades))
+            if any(x["timestamp"] < start_timestamp for x in trades):
+                continue
+            if any(x["side"] != self.action for x in trades):
+                continue
+            if any(x["info"]["category"] == "exchange" and self.account == "margin" for x in trades) or\
+                    any(x["info"]["category"] == "marginTrade" and self.account == "exchange" for x in trades):
+                continue
+            trade_sum = sum(D(x["info"]["amount"]) for x in trades)
+            amount_diff = round(abs(trade_sum - amount),
+                    self.market.ccxt.order_precision(symbol))
+            if amount_diff != 0:
+                continue
+            if (self.action == "sell" and any(D(x["info"]["rate"]) < self.rate for x in trades)) or\
+                    (self.action == "buy" and any(D(x["info"]["rate"]) > self.rate for x in trades)):
+                continue
+            self.results.append({"id": order_id})
+            return True
+
+        return False
 
 class Mouvement:
     def __init__(self, currency, base_currency, hash_):
@@ -629,7 +686,7 @@ class Mouvement:
         self.action = hash_.get("type")
         self.fee_rate = D(hash_.get("fee", -1))
         try:
-            self.date = datetime.strptime(hash_.get("date", ""), '%Y-%m-%d %H:%M:%S')
+            self.date = datetime.datetime.strptime(hash_.get("date", ""), '%Y-%m-%d %H:%M:%S')
         except ValueError:
             self.date = None
         self.rate = D(hash_.get("rate", 0))