Implement order following

- Change urllib to requests and use it in tests
- Add debug mode for sensitive actions
- Follow orders when trade is running
- Fetch order from the api and store the mouvements already filled
- Start rewriting tests

1 files changed, 175 insertions, 55 deletions

diff --git a/portfolio.py b/portfolio.py
index 0ab16fd..b3065b8 100644
--- a/portfolio.py
+++ b/portfolio.py
@@ -3,6 +3,11 @@ import time
 from decimal import Decimal as D, ROUND_DOWN
 # Put your poloniex api key in market.py
 from market import market
+from json import JSONDecodeError
+import requests
+
+# FIXME: correctly handle web call timeouts
+
 
 class Portfolio:
     URL = "https://cryptoportfolio.io/wp-content/uploads/portfolio/json/cryptoportfolio.json"
@@ -18,20 +23,13 @@ class Portfolio:
 
     @classmethod
     def get_cryptoportfolio(cls):
-        import json
-        import urllib3
-        urllib3.disable_warnings()
-        http = urllib3.PoolManager()
-
         try:
-            r = http.request("GET", cls.URL)
+            r = requests.get(cls.URL)
         except Exception:
-            return None
+            return
         try:
-            cls.data = json.loads(r.data,
-                    parse_int=D,
-                    parse_float=D)
-        except json.JSONDecodeError:
+            cls.data = r.json(parse_int=D, parse_float=D)
+        except JSONDecodeError:
             cls.data = None
 
     @classmethod
@@ -79,9 +77,6 @@ class Amount:
         self.ticker = ticker
         self.rate = rate
 
-        self.ticker_cache = {}
-        self.ticker_cache_timestamp = time.time()
-
     def in_currency(self, other_currency, market, rate=None, action=None, compute_value="average"):
         if other_currency == self.currency:
             return self
@@ -141,9 +136,6 @@ class Amount:
     def __truediv__(self, value):
         return self.__floordiv__(value)
 
-    def __le__(self, other):
-        return self == other or self < other
-
     def __lt__(self, other):
         if other == 0:
             return self.value < 0
@@ -151,6 +143,9 @@ class Amount:
             raise Exception("Comparing amounts must be done with same currencies")
         return self.value < other.value
 
+    def __le__(self, other):
+        return self == other or self < other
+
     def __gt__(self, other):
         return not self <= other
 
@@ -192,9 +187,9 @@ class Balance:
                 "margin_total", "margin_borrowed", "margin_free"]:
             setattr(self, key, Amount(currency, hash_.get(key, 0)))
 
-        self.margin_position_type = hash_["margin_position_type"]
+        self.margin_position_type = hash_.get("margin_position_type")
 
-        if hash_["margin_borrowed_base_currency"] is not None:
+        if hash_.get("margin_borrowed_base_currency") is not None:
             base_currency = hash_["margin_borrowed_base_currency"]
             for key in [
                     "margin_liquidation_price",
@@ -227,7 +222,6 @@ class Balance:
                 cls.known_balances[currency] = cls(currency, balance)
         return cls.known_balances
 
-
     @classmethod
     def dispatch_assets(cls, amount, repartition=None):
         if repartition is None:
@@ -239,34 +233,34 @@ class Balance:
             if trade_type == "short":
                 amounts[currency] = - amounts[currency]
             if currency not in cls.known_balances:
-                cls.known_balances[currency] = cls(currency, 0, 0, 0)
+                cls.known_balances[currency] = cls(currency, {})
         return amounts
 
     @classmethod
-    def prepare_trades(cls, market, base_currency="BTC", compute_value="average"):
+    def prepare_trades(cls, market, base_currency="BTC", compute_value="average", debug=False):
         cls.fetch_balances(market)
         values_in_base = cls.in_currency(base_currency, market, compute_value=compute_value)
         total_base_value = sum(values_in_base.values())
         new_repartition = cls.dispatch_assets(total_base_value)
         # Recompute it in case we have new currencies
         values_in_base = cls.in_currency(base_currency, market, compute_value=compute_value)
-        Trade.compute_trades(values_in_base, new_repartition, market=market)
+        Trade.compute_trades(values_in_base, new_repartition, market=market, debug=debug)
 
     @classmethod
-    def update_trades(cls, market, base_currency="BTC", compute_value="average", only=None):
+    def update_trades(cls, market, base_currency="BTC", compute_value="average", only=None, debug=False):
         cls.fetch_balances(market)
         values_in_base = cls.in_currency(base_currency, market, compute_value=compute_value)
         total_base_value = sum(values_in_base.values())
         new_repartition = cls.dispatch_assets(total_base_value)
-        Trade.compute_trades(values_in_base, new_repartition, only=only, market=market)
+        Trade.compute_trades(values_in_base, new_repartition, only=only, market=market, debug=debug)
 
     @classmethod
-    def prepare_trades_to_sell_all(cls, market, base_currency="BTC", compute_value="average"):
+    def prepare_trades_to_sell_all(cls, market, base_currency="BTC", compute_value="average", debug=False):
         cls.fetch_balances(market)
         values_in_base = cls.in_currency(base_currency, market, compute_value=compute_value)
         total_base_value = sum(values_in_base.values())
         new_repartition = cls.dispatch_assets(total_base_value, repartition={ base_currency: (1, "long") })
-        Trade.compute_trades(values_in_base, new_repartition, market=market)
+        Trade.compute_trades(values_in_base, new_repartition, market=market, debug=debug)
 
     def __repr__(self):
         if self.exchange_total > 0:
@@ -309,6 +303,7 @@ class Computation:
             }
 
 class Trade:
+    debug = False
     trades = []
 
     def __init__(self, value_from, value_to, currency, market=None):
@@ -370,7 +365,8 @@ class Trade:
         return cls.get_ticker(c1, c2, market)
 
     @classmethod
-    def compute_trades(cls, values_in_base, new_repartition, only=None, market=None):
+    def compute_trades(cls, values_in_base, new_repartition, only=None, market=None, debug=False):
+        cls.debug = cls.debug or debug
         base_currency = sum(values_in_base.values()).currency
         for currency in Balance.currencies():
             if currency == base_currency:
@@ -402,7 +398,7 @@ class Trade:
                 trade.prepare_order(compute_value=compute_value)
 
     @classmethod
-    def move_balances(cls, market, debug=False):
+    def move_balances(cls, market):
         needed_in_margin = {} 
         for trade in cls.trades:
             if trade.trade_type == "short":
@@ -414,12 +410,12 @@ class Trade:
             delta = (needed - current_balance).value
             # FIXME: don't remove too much if there are open margin position
             if delta > 0:
-                if debug:
+                if cls.debug:
                     print("market.transfer_balance({}, {}, 'exchange', 'margin')".format(currency, delta))
                 else:
                     market.transfer_balance(currency, delta, "exchange", "margin")
             elif delta < 0:
-                if debug:
+                if cls.debug:
                     print("market.transfer_balance({}, {}, 'margin', 'exchange')".format(currency, -delta))
                 else:
                     market.transfer_balance(currency, -delta, "margin", "exchange")
@@ -449,6 +445,37 @@ class Trade:
         else:
             return "long"
 
+    @property
+    def filled_amount(self):
+        filled_amount = 0
+        for order in self.orders:
+            filled_amount += order.filled_amount
+        return filled_amount
+
+    def update_order(self, order, tick):
+        new_order = None
+        if tick in [0, 1, 3, 4, 6]:
+            print("{}, tick {}, waiting".format(order, tick))
+        elif tick == 2:
+            self.prepare_order(compute_value=lambda x, y: (x[y] + x["average"]) / 2)
+            new_order = self.orders[-1]
+            print("{}, tick {}, cancelling and adjusting to {}".format(order, tick, new_order))
+        elif tick ==5:
+            self.prepare_order(compute_value=lambda x, y: (x[y]*2 + x["average"]) / 3)
+            new_order = self.orders[-1]
+            print("{}, tick {}, cancelling and adjusting to {}".format(order, tick, new_order))
+        elif tick >= 7:
+            if tick == 7:
+                print("{}, tick {}, fallbacking to market value".format(order, tick))
+            if (tick - 7) % 3 == 0:
+                self.prepare_order(compute_value="default")
+                new_order = self.orders[-1]
+                print("{}, tick {}, market value, cancelling and adjusting to {}".format(order, tick, new_order))
+
+        if new_order is not None:
+            order.cancel()
+            new_order.run()
+
     def prepare_order(self, compute_value="default"):
         if self.action is None:
             return
@@ -492,12 +519,20 @@ class Trade:
             # buy:
             #   I want to buy 9 / 0.1 FOO
             #   Action: "sell" "9 BTC" at rate "1/0.1" "FOO" on "market"
+            if self.value_to == 0:
+                rate = self.value_from.linked_to.value / self.value_from.value
+                # Recompute the rate to avoid any rounding error
 
         close_if_possible = (self.value_to == 0)
 
+        if delta <= self.filled_amount:
+            print("Less to do than already filled: {} <= {}".format(delta,
+                self.filled_amount))
+            return
+
         self.orders.append(Order(self.order_action(inverted),
-            delta, rate, currency, self.trade_type, self.market,
-            close_if_possible=close_if_possible))
+            delta - self.filled_amount, rate, currency, self.trade_type,
+            self.market, self, close_if_possible=close_if_possible))
 
     @classmethod
     def compute_value(cls, ticker, action, compute_value="default"):
@@ -523,18 +558,19 @@ class Trade:
             order.run()
 
     @classmethod
-    def follow_orders(cls, verbose=True, sleep=30):
-        orders = cls.all_orders()
-        finished_orders = []
-        while len(orders) != len(finished_orders):
+    def follow_orders(cls, verbose=True, sleep=None):
+        if sleep is None:
+            sleep = 7 if cls.debug else 30
+        tick = 0
+        while len(cls.all_orders(state="open")) > 0:
             time.sleep(sleep)
-            for order in orders:
-                if order in finished_orders:
-                    continue
+            tick += 1
+            for order in cls.all_orders(state="open"):
                 if order.get_status() != "open":
-                    finished_orders.append(order)
                     if verbose:
                         print("finished {}".format(order))
+                else:
+                    order.trade.update_order(order, tick)
         if verbose:
             print("All orders finished")
 
@@ -562,16 +598,19 @@ class Trade:
 
 class Order:
     def __init__(self, action, amount, rate, base_currency, trade_type, market,
-            close_if_possible=False):
+            trade, close_if_possible=False):
         self.action = action
         self.amount = amount
         self.rate = rate
         self.base_currency = base_currency
         self.market = market
         self.trade_type = trade_type
-        self.result = None
+        self.results = []
+        self.mouvements = []
         self.status = "pending"
+        self.trade = trade
         self.close_if_possible = close_if_possible
+        self.debug = trade.debug
 
     def __repr__(self):
         return "Order({} {} {} at {} {} [{}]{})".format(
@@ -599,16 +638,22 @@ class Order:
     def finished(self):
         return self.status == "closed" or self.status == "canceled" or self.status == "error"
 
-    def run(self, debug=False):
+    @property
+    def id(self):
+        return self.results[0]["id"]
+
+    def run(self):
         symbol = "{}/{}".format(self.amount.currency, self.base_currency)
         amount = round(self.amount, self.market.order_precision(symbol)).value
 
-        if debug:
+        if self.debug:
             print("market.create_order('{}', 'limit', '{}', {}, price={}, account={})".format(
                 symbol, self.action, amount, self.rate, self.account))
+            self.status = "open"
+            self.results.append({"debug": True, "id": -1})
         else:
             try:
-                self.result = self.market.create_order(symbol, 'limit', self.action, amount, price=self.rate, account=self.account)
+                self.results.append(self.market.create_order(symbol, 'limit', self.action, amount, price=self.rate, account=self.account))
                 self.status = "open"
             except Exception as e:
                 self.status = "error"
@@ -618,22 +663,84 @@ class Order:
                 print(self.error_message)
 
     def get_status(self):
+        if self.debug:
+            return self.status
         # other states are "closed" and "canceled"
         if self.status == "open":
-            result = self.market.fetch_order(self.result['id'])
-            if result["status"] != "open":
-                self.mark_finished_order(result["status"])
+            self.fetch()
+            if self.status != "open":
+                self.mark_finished_order()
         return self.status
 
-    def mark_finished_order(self, status):
-        if status == "closed":
+    def mark_finished_order(self):
+        if self.debug:
+            return
+        if self.status == "closed":
             if self.trade_type == "short" and self.action == "buy" and self.close_if_possible:
                 self.market.close_margin_position(self.amount.currency, self.base_currency)
 
+    fetch_cache_timestamp = None
+    def fetch(self, force=False):
+        if self.debug or (not force and self.fetch_cache_timestamp is not None
+                and time.time() - self.fetch_cache_timestamp < 10):
+            return
+        self.fetch_cache_timestamp = time.time()
+
+        self.results.append(self.market.fetch_order(self.id))
+        result = self.results[-1]
         self.status = result["status"]
+        # Time at which the order started
+        self.timestamp = result["datetime"]
+        self.fetch_mouvements()
+
+        # FIXME: consider open order with dust remaining as closed
+
+    @property
+    def dust_amount_remaining(self):
+        return self.remaining_amount < 0.001
+
+    @property
+    def remaining_amount(self):
+        if self.status == "open":
+            self.fetch()
+        return self.amount - self.filled_amount
+
+    @property
+    def filled_amount(self):
+        if self.status == "open":
+            self.fetch()
+        filled_amount = Amount(self.amount.currency, 0)
+        for mouvement in self.mouvements:
+            filled_amount += mouvement.total
+        return filled_amount
+
+    def fetch_mouvements(self):
+        mouvements = self.market.privatePostReturnOrderTrades({"orderNumber": self.id})
+        self.mouvements = []
+
+        for mouvement_hash in mouvements:
+            self.mouvements.append(Mouvement(self.amount.currency,
+                self.base_currency, mouvement_hash))
 
     def cancel(self):
+        if self.debug:
+            self.status = "canceled"
+            return
         self.market.cancel_order(self.result['id'])
+        self.fetch()
+
+class Mouvement:
+    def __init__(self, currency, base_currency, hash_):
+        self.currency = currency
+        self.base_currency = base_currency
+        self.id = hash_["id"]
+        self.action = hash_["type"]
+        self.fee_rate = D(hash_["fee"])
+        self.date = datetime.strptime(hash_["date"], '%Y-%m-%d %H:%M:%S')
+        self.rate = D(hash_["rate"])
+        self.total = Amount(currency, hash_["amount"])
+        # rate * total = total_in_base
+        self.total_in_base = Amount(base_currency, hash_["total"])
 
 def print_orders(market, base_currency="BTC"):
     Balance.prepare_trades(market, base_currency=base_currency, compute_value="average")
@@ -650,14 +757,27 @@ def make_orders(market, base_currency="BTC"):
             print("\t", order, sep="")
             order.run()
 
-def sell_all(market, base_currency="BTC"):
-    Balance.prepare_trades_to_sell_all(market)
+def process_sell_all_sell(market, base_currency="BTC", debug=False):
+    Balance.prepare_trades_to_sell_all(market, debug=debug)
     Trade.prepare_orders(compute_value="average")
+    print("------------------")
+    for currency, balance in Balance.known_balances.items():
+        print(balance)
+    print("------------------")
+    Trade.print_all_with_order()
+    print("------------------")
     Trade.run_orders()
     Trade.follow_orders()
 
-    Balance.update_trades(market, only="acquire")
-    Trade.prepare_orders(only="acquire")
+def process_sell_all_buy(market, base_currency="BTC", debug=False):
+    Balance.prepare_trades(market, debug=debug)
+    Trade.prepare_orders()
+    print("------------------")
+    for currency, balance in Balance.known_balances.items():
+        print(balance)
+    print("------------------")
+    Trade.print_all_with_order()
+    print("------------------")
     Trade.move_balances(market)
     Trade.run_orders()
     Trade.follow_orders()