Various fixes/improvementsv0.3

- Use pending gains to compute the move_balance
- Use ttl_cache for tickers

6 files changed, 394 insertions, 245 deletions

diff --git a/ccxt_wrapper.py b/ccxt_wrapper.py
index b79cd37..903bfc4 100644
--- a/ccxt_wrapper.py
+++ b/ccxt_wrapper.py
@@ -38,12 +38,13 @@ class poloniexE(poloniex):
         """
         portfolio.market.privatePostGetMarginPosition({"currencyPair": "BTC_DASH"})
         See DASH/BTC positions
-        {'amount': '-0.10000000',          -> DASH empruntés
+        {'amount': '-0.10000000',         -> DASH empruntés (à rendre)
         'basePrice': '0.06818560',        -> à ce prix là (0.06828800 demandé * (1-0.15%))
-        'lendingFees': '0.00000000',      -> ce que je dois à mon créditeur
+        'lendingFees': '0.00000000',      -> ce que je dois à mon créditeur en intérêts
         'liquidationPrice': '0.15107132', -> prix auquel ça sera liquidé (dépend de ce que j’ai déjà sur mon compte margin)
         'pl': '-0.00000371',              -> plus-value latente si je rachète tout de suite (négatif = perdu)
-        'total': '0.00681856',            -> valeur totale empruntée en BTC
+        'total': '0.00681856',            -> valeur totale empruntée en BTC (au moment de l'échange)
+                                             = amount * basePrice à erreur d'arrondi près
         'type': 'short'}
         """
         positions = self.privatePostGetMarginPosition({"currencyPair": "all"})
@@ -70,8 +71,7 @@ class poloniexE(poloniex):
         for key, balance in balances.items():
             result[key] = {}
             for currency, amount in balance.items():
-                if currency not in result:
-                    result[currency] = {}
+                result.setdefault(currency, {})
                 result[currency][key] = decimal.Decimal(amount)
                 result[key][currency] = decimal.Decimal(amount)
         return result
@@ -83,6 +83,7 @@ class poloniexE(poloniex):
 
         all_balances = {}
         in_positions = {}
+        pending_pl = {}
 
         for currency, exchange_balance in exchange_balances.items():
             if currency in ["info", "free", "used", "total"]:
@@ -96,25 +97,106 @@ class poloniexE(poloniex):
                 "exchange_used": exchange_balance["used"],
                 "exchange_total": exchange_balance["total"] - balance_per_type.get("margin", 0),
                 "exchange_free": exchange_balance["free"] - balance_per_type.get("margin", 0),
-                "margin_free": balance_per_type.get("margin", 0) + margin_balance.get("amount", 0),
-                "margin_borrowed": 0,
+                # Disponible sur le compte margin
+                "margin_available": balance_per_type.get("margin", 0),
+                # Bloqué en position
+                "margin_in_position": 0,
+                # Emprunté
+                "margin_borrowed": -margin_balance.get("amount", 0),
+                # Total
                 "margin_total": balance_per_type.get("margin", 0) + margin_balance.get("amount", 0),
+                "margin_pending_gain": 0,
                 "margin_lending_fees": margin_balance.get("lendingFees", 0),
-                "margin_pending_gain": margin_balance.get("pl", 0),
+                "margin_pending_base_gain": margin_balance.get("pl", 0),
                 "margin_position_type": margin_balance.get("type", None),
                 "margin_liquidation_price": margin_balance.get("liquidationPrice", 0),
                 "margin_borrowed_base_price": margin_balance.get("total", 0),
                 "margin_borrowed_base_currency": margin_balance.get("baseCurrency", None),
                 }
             if len(margin_balance) > 0:
-                if margin_balance["baseCurrency"] not in in_positions:
-                    in_positions[margin_balance["baseCurrency"]] = 0
+                in_positions.setdefault(margin_balance["baseCurrency"], 0)
                 in_positions[margin_balance["baseCurrency"]] += margin_balance["total"]
 
+                pending_pl.setdefault(margin_balance["baseCurrency"], 0)
+                pending_pl[margin_balance["baseCurrency"]] += margin_balance["pl"]
+
+            # J’emprunte 0.12062983 que je revends à 0.06003598 BTC/DASH, soit 0.00724213 BTC.
+            # Sur ces 0.00724213 BTC je récupère 0.00724213*(1-0.0015) = 0.00723127 BTC
+            # 
+            # -> ordertrades ne tient pas compte des fees
+            #     amount = montant vendu (un seul mouvement)
+            #     rate = à ce taux
+            #     total = total en BTC (pour ce mouvement)
+            # -> marginposition:
+            #     amount = ce que je dois rendre
+            #     basePrice = prix de vente en tenant compte des fees
+            #     (amount * basePrice = la quantité de BTC que j’ai effectivement
+            #     reçue à erreur d’arrondi près, utiliser plutôt "total")
+            #     total = la quantité de BTC que j’ai reçue
+            #     pl = plus value actuelle si je rachetais tout de suite
+            # -> marginaccountsummary:
+            #     currentMargin = La marge actuelle (= netValue/totalBorrowedValue)
+            #     totalValue = BTC actuellement en margin (déposé)
+            #     totalBorrowedValue = sum (amount * ticker[lowestAsk])
+            #     pl = sum(pl)
+            #     netValue = BTC actuellement en margin (déposé) + pl
+            # Exemple:
+            # In [38]: m.ccxt.private_post_returnordertrades({"orderNumber": "XXXXXXXXXXXX"})
+            # Out[38]:
+            # [{'amount': '0.11882982',
+            #   'currencyPair': 'BTC_DASH',
+            #   'date': '2018-02-26 22:48:35',
+            #   'fee': '0.00150000',
+            #   'globalTradeID': 348891380,
+            #   'rate': '0.06003598',
+            #   'total': '0.00713406',
+            #   'tradeID': 9634443,
+            #   'type': 'sell'},
+            #  {'amount': '0.00180000',
+            #   'currencyPair': 'BTC_DASH',
+            #   'date': '2018-02-26 22:48:30',
+            #   'fee': '0.00150000',
+            #   'globalTradeID': 348891375,
+            #   'rate': '0.06003598',
+            #   'total': '0.00010806',
+            #   'tradeID': 9634442,
+            #   'type': 'sell'}]
+            # 
+            # In [51]: m.ccxt.privatePostGetMarginPosition({"currencyPair": "BTC_DASH"})
+            # Out[51]:
+            # {'amount': '-0.12062982',
+            #  'basePrice': '0.05994587',
+            #  'lendingFees': '0.00000000',
+            #  'liquidationPrice': '0.15531479',
+            #  'pl': '0.00000122',
+            #  'total': '0.00723126',
+            #  'type': 'short'}
+            # In [52]: m.ccxt.privatePostGetMarginPosition({"currencyPair": "BTC_BTS"})
+            # Out[52]:
+            # {'amount': '-332.97159188',
+            #  'basePrice': '0.00002171',
+            #  'lendingFees': '0.00000000',
+            #  'liquidationPrice': '0.00005543',
+            #  'pl': '0.00029548',
+            #  'total': '0.00723127',
+            #  'type': 'short'}
+            # 
+            # In [53]: m.ccxt.privatePostReturnMarginAccountSummary()
+            # Out[53]:
+            # {'currentMargin': '1.04341991',
+            #  'lendingFees': '0.00000000',
+            #  'netValue': '0.01478093',
+            #  'pl': '0.00029666',
+            #  'totalBorrowedValue': '0.01416585',
+            #  'totalValue': '0.01448427'}
+
         for currency, in_position in in_positions.items():
             all_balances[currency]["total"] += in_position
+            all_balances[currency]["margin_in_position"] += in_position
             all_balances[currency]["margin_total"] += in_position
-            all_balances[currency]["margin_borrowed"] += in_position
+
+        for currency, pl in pending_pl.items():
+            all_balances[currency]["margin_pending_gain"] += pl
 
         return all_balances
 
@@ -228,8 +310,9 @@ class poloniexE(poloniex):
         'lendingFees': '0.00000000',        -> fees totaux
         'netValue': '0.01008254',           -> balance + plus-value
         'pl': '0.00008254',                 -> plus value latente (somme des positions)
-        'totalBorrowedValue': '0.00673602', -> valeur en BTC empruntée
-        'totalValue': '0.01000000'}         -> valeur totale en compte
+        'totalBorrowedValue': '0.00673602', -> valeur empruntée convertie en BTC.
+                (= sum(amount * ticker[lowerAsk]) pour amount dans marginposition)
+        'totalValue': '0.01000000'}         -> balance (collateral déposé en margin)
         """
         summary = self.privatePostReturnMarginAccountSummary()
 
diff --git a/market.py b/market.py
index 0cb3e67..9122fc0 100644
--- a/market.py
+++ b/market.py
@@ -1,7 +1,8 @@
-from ccxt import ExchangeError
+from ccxt import ExchangeError, NotSupported
 import ccxt_wrapper as ccxt
 import time
 from store import *
+from cachetools.func import ttl_cache
 
 class Market:
     debug = False
@@ -42,19 +43,17 @@ class Market:
         needed_in_margin = {} 
         moving_to_margin = {}
 
-        for currency in self.balances.all:
-            if self.balances.all[currency].margin_free != 0:
-                needed_in_margin[currency] = 0
-        for trade in self.trades.all:
-            if trade.value_to.currency not in needed_in_margin:
-                needed_in_margin[trade.value_to.currency] = 0
+        for currency, balance in self.balances.all.items():
+            needed_in_margin[currency] = balance.margin_in_position - balance.margin_pending_gain
+        for trade in self.trades.pending:
+            needed_in_margin.setdefault(trade.base_currency, 0)
             if trade.trade_type == "short":
-                needed_in_margin[trade.value_to.currency] += abs(trade.value_to)
+                needed_in_margin[trade.base_currency] -= trade.delta
         for currency, needed in needed_in_margin.items():
-            current_balance = self.balances.all[currency].margin_free
+            current_balance = self.balances.all[currency].margin_available
             moving_to_margin[currency] = (needed - current_balance)
             delta = moving_to_margin[currency].value
-            if self.debug:
+            if self.debug and delta != 0:
                 self.report.log_debug_action("Moving {} from exchange to margin".format(moving_to_margin[currency]))
                 continue
             if delta > 0:
@@ -65,14 +64,18 @@ class Market:
 
         self.balances.fetch_balances()
 
-    fees_cache = {}
+    @ttl_cache(ttl=3600)
     def fetch_fees(self):
-        if self.ccxt.__class__ not in self.fees_cache:
-            self.fees_cache[self.ccxt.__class__] = self.ccxt.fetch_fees()
-        return self.fees_cache[self.ccxt.__class__]
+        return self.ccxt.fetch_fees()
+
+    @ttl_cache(maxsize=20, ttl=5)
+    def get_tickers(self, refresh=False):
+        try:
+            return self.ccxt.fetch_tickers()
+        except NotSupported:
+            return None
 
-    ticker_cache = {}
-    ticker_cache_timestamp = time.time()
+    @ttl_cache(maxsize=20, ttl=5)
     def get_ticker(self, c1, c2, refresh=False):
         def invert(ticker):
             return {
@@ -86,25 +89,25 @@ class Market:
                 "average": (ticker["bid"] + ticker["ask"] ) / 2,
                 })
 
-        if time.time() - self.ticker_cache_timestamp > 5:
-            self.ticker_cache = {}
-            self.ticker_cache_timestamp = time.time()
-        elif not refresh:
-            if (c1, c2, self.ccxt.__class__) in self.ticker_cache:
-                return self.ticker_cache[(c1, c2, self.ccxt.__class__)]
-            if (c2, c1, self.ccxt.__class__) in self.ticker_cache:
-                return invert(self.ticker_cache[(c2, c1, self.ccxt.__class__)])
-
-        try:
-            self.ticker_cache[(c1, c2, self.ccxt.__class__)] = self.ccxt.fetch_ticker("{}/{}".format(c1, c2))
-            augment_ticker(self.ticker_cache[(c1, c2, self.ccxt.__class__)])
-        except ExchangeError:
+        tickers = self.get_tickers()
+        if tickers is None:
             try:
-                self.ticker_cache[(c2, c1, self.ccxt.__class__)] = self.ccxt.fetch_ticker("{}/{}".format(c2, c1))
-                augment_ticker(self.ticker_cache[(c2, c1, self.ccxt.__class__)])
+                ticker = self.ccxt.fetch_ticker("{}/{}".format(c1, c2))
+                augment_ticker(ticker)
             except ExchangeError:
-                self.ticker_cache[(c1, c2, self.ccxt.__class__)] = None
-        return self.get_ticker(c1, c2)
+                try:
+                    ticker = invert(self.ccxt.fetch_ticker("{}/{}".format(c2, c1)))
+                except ExchangeError:
+                    ticker = None
+        else:
+            if "{}/{}".format(c1, c2) in tickers:
+                ticker = tickers["{}/{}".format(c1, c2)]
+                augment_ticker(ticker)
+            elif "{}/{}".format(c2, c1) in tickers:
+                ticker = invert(tickers["{}/{}".format(c2, c1)])
+            else:
+                ticker = None
+        return ticker
 
     def follow_orders(self, sleep=None):
         if sleep is None:
diff --git a/portfolio.py b/portfolio.py
index eb3390e..b77850b 100644
--- a/portfolio.py
+++ b/portfolio.py
@@ -3,7 +3,7 @@ from datetime import datetime, timedelta
 from decimal import Decimal as D, ROUND_DOWN
 from json import JSONDecodeError
 from simplejson.errors import JSONDecodeError as SimpleJSONDecodeError
-from ccxt import ExchangeError, InsufficientFunds, ExchangeNotAvailable, InvalidOrder
+from ccxt import ExchangeError, InsufficientFunds, ExchangeNotAvailable, InvalidOrder, OrderNotCached
 from retry import retry
 import requests
 
@@ -226,8 +226,8 @@ class Amount:
 
 class Balance:
     base_keys = ["total", "exchange_total", "exchange_used",
-            "exchange_free", "margin_total", "margin_borrowed",
-            "margin_free"]
+            "exchange_free", "margin_total", "margin_in_position",
+            "margin_available", "margin_borrowed", "margin_pending_gain"]
 
     def __init__(self, currency, hash_):
         self.currency = currency
@@ -240,8 +240,8 @@ class Balance:
             base_currency = hash_["margin_borrowed_base_currency"]
             for key in [
                     "margin_liquidation_price",
-                    "margin_pending_gain",
                     "margin_lending_fees",
+                    "margin_pending_base_gain",
                     "margin_borrowed_base_price"
                     ]:
                 setattr(self, key, Amount(base_currency, hash_.get(key, 0)))
@@ -261,12 +261,12 @@ class Balance:
             exchange = ""
 
         if self.margin_total > 0:
-            if self.margin_free != 0 and self.margin_borrowed != 0:
-                margin = " Margin: [✔{} + borrowed {} = {}]".format(str(self.margin_free), str(self.margin_borrowed), str(self.margin_total))
-            elif self.margin_free != 0:
-                margin = " Margin: [✔{}]".format(str(self.margin_free))
+            if self.margin_available != 0 and self.margin_in_position != 0:
+                margin = " Margin: [✔{} + ❌{} = {}]".format(str(self.margin_available), str(self.margin_in_position), str(self.margin_total))
+            elif self.margin_available != 0:
+                margin = " Margin: [✔{}]".format(str(self.margin_available))
             else:
-                margin = " Margin: [borrowed {}]".format(str(self.margin_borrowed))
+                margin = " Margin: [❌{}]".format(str(self.margin_in_position))
         elif self.margin_total < 0:
             margin = " Margin: [{} @@ {}/{}]".format(str(self.margin_total),
                     str(self.margin_borrowed_base_price),
@@ -290,6 +290,7 @@ class Trade:
         self.value_to = value_to
         self.orders = []
         self.market = market
+        assert self.value_from.value * self.value_to.value >= 0
         assert self.value_from.currency == self.value_to.currency
         if self.value_from != 0:
             assert self.value_from.linked_to is not None and self.value_from.linked_to.currency == self.currency
@@ -298,6 +299,10 @@ class Trade:
         self.base_currency = self.value_from.currency
 
     @property
+    def delta(self):
+        return self.value_to - self.value_from
+
+    @property
     def action(self):
         if self.value_from == self.value_to:
             return None
@@ -322,6 +327,10 @@ class Trade:
         else:
             return "long"
 
+    @property
+    def is_fullfiled(self):
+        return abs(self.filled_amount(in_base_currency=True)) >= abs(self.delta)
+
     def filled_amount(self, in_base_currency=False):
         filled_amount = 0
         for order in self.orders:
@@ -329,34 +338,36 @@ class Trade:
         return filled_amount
 
     def update_order(self, order, tick):
-        new_order = None
-        if tick in [0, 1, 3, 4, 6]:
+        actions = {
+                0: ["waiting", None],
+                1: ["waiting", None],
+                2: ["adjusting", lambda x, y: (x[y] + x["average"]) / 2],
+                3: ["waiting", None],
+                4: ["waiting", None],
+                5: ["adjusting", lambda x, y: (x[y]*2 + x["average"]) / 3],
+                6: ["waiting", None],
+                7: ["market_fallback", "default"],
+                }
+
+        if tick in actions:
+            update, compute_value = actions[tick]
+        elif tick % 3 == 1:
+            update = "market_adjust"
+            compute_value = "default"
+        else:
             update = "waiting"
             compute_value = None
-        elif tick == 2:
-            update = "adjusting"
-            compute_value = 'lambda x, y: (x[y] + x["average"]) / 2'
-            new_order = self.prepare_order(compute_value=lambda x, y: (x[y] + x["average"]) / 2)
-        elif tick ==5:
-            update = "adjusting"
-            compute_value = 'lambda x, y: (x[y]*2 + x["average"]) / 3'
-            new_order = self.prepare_order(compute_value=lambda x, y: (x[y]*2 + x["average"]) / 3)
-        elif tick >= 7:
-            if (tick - 7) % 3 == 0:
-                new_order = self.prepare_order(compute_value="default")
-                update = "market_adjust"
-                compute_value = "default"
-            else:
-                update = "waiting"
-                compute_value = None
-            if tick == 7:
-                update = "market_fallback"
+
+        if compute_value is not None:
+            order.cancel()
+            new_order = self.prepare_order(compute_value=compute_value)
+        else:
+            new_order = None
 
         self.market.report.log_order(order, tick, update=update,
                 compute_value=compute_value, new_order=new_order)
 
         if new_order is not None:
-            order.cancel()
             new_order.run()
             self.market.report.log_order(order, tick, new_order=new_order)
 
@@ -369,10 +380,9 @@ class Trade:
             ticker = ticker["original"]
         rate = Computation.compute_value(ticker, self.order_action(inverted), compute_value=compute_value)
 
-        #TODO: store when the order is considered filled
         # FIXME: Dust amount should be removed from there if they werent
         # honored in other sales
-        delta_in_base = abs(self.value_from - self.value_to)
+        delta_in_base = abs(self.delta)
         # 9 BTC's worth of move (10 - 1 or 1 - 10 depending on case)
 
         if not inverted:
@@ -477,7 +487,6 @@ class Order:
         self.trade = trade
         self.close_if_possible = close_if_possible
         self.id = None
-        self.fetch_cache_timestamp = None
         self.tries = 0
 
     def as_json(self):
@@ -578,21 +587,19 @@ class Order:
             if self.trade_type == "short" and self.action == "buy" and self.close_if_possible:
                 self.market.ccxt.close_margin_position(self.amount.currency, self.base_currency)
 
-    def fetch(self, force=False):
+    def fetch(self):
         if self.market.debug:
             self.market.report.log_debug_action("Fetching {}".format(self))
             return
-        if (not force and self.fetch_cache_timestamp is not None
-                and time.time() - self.fetch_cache_timestamp < 10):
-            return
-        self.fetch_cache_timestamp = time.time()
-
-        result = self.market.ccxt.fetch_order(self.id, symbol=self.amount.currency)
-        self.results.append(result)
+        try:
+            result = self.market.ccxt.fetch_order(self.id, symbol=self.amount.currency)
+            self.results.append(result)
+            self.status = result["status"]
+            # Time at which the order started
+            self.timestamp = result["datetime"]
+        except OrderNotCached:
+            self.status = "closed_unknown"
 
-        self.status = result["status"]
-        # Time at which the order started
-        self.timestamp = result["datetime"]
         self.fetch_mouvements()
 
         # FIXME: consider open order with dust remaining as closed
@@ -601,8 +608,6 @@ class Order:
         return self.remaining_amount() < Amount(self.amount.currency, D("0.001"))
 
     def remaining_amount(self):
-        if self.status == "open":
-            self.fetch()
         return self.amount - self.filled_amount()
 
     def filled_amount(self, in_base_currency=False):
@@ -633,7 +638,7 @@ class Order:
             self.status = "canceled"
             return
         self.market.ccxt.cancel_order(self.id)
-        self.fetch(force=True)
+        self.fetch()
 
 class Mouvement:
     def __init__(self, currency, base_currency, hash_):
diff --git a/requirements.txt b/requirements.txt
index 5a25dbf..1bc76ec 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@ requests>=2.18.4
 requests_mock==1.4.0
 psycopg2==2.7.4
 retry==0.9.2
+cachetools==2.0.1
diff --git a/store.py b/store.py
index bebd127..c581608 100644
--- a/store.py
+++ b/store.py
@@ -2,6 +2,7 @@ import portfolio
 import simplejson as json
 from decimal import Decimal as D, ROUND_DOWN
 from datetime import date, datetime
+import inspect
 
 __all__ = ["BalanceStore", "ReportStore", "TradeStore"]
 
@@ -55,6 +56,9 @@ class ReportStore:
             compute_value, type):
         values = {}
         rates = {}
+        if callable(compute_value):
+            compute_value = inspect.getsource(compute_value).strip()
+
         for currency, amount in amounts.items():
             values[currency] = amount.as_json()["value"]
             rates[currency] = amount.rate
@@ -92,6 +96,8 @@ class ReportStore:
             })
 
     def log_orders(self, orders, tick=None, only=None, compute_value=None):
+        if callable(compute_value):
+            compute_value = inspect.getsource(compute_value).strip()
         self.print_log("[Orders]")
         self.market.trades.print_all_with_order(ind="\t")
         self.add_log({
@@ -104,6 +110,8 @@ class ReportStore:
 
     def log_order(self, order, tick, finished=False, update=None,
             new_order=None, compute_value=None):
+        if callable(compute_value):
+            compute_value = inspect.getsource(compute_value).strip()
         if finished:
             self.print_log("[Order] Finished {}".format(order))
         elif update == "waiting":
@@ -201,8 +209,7 @@ class BalanceStore:
             amounts[currency] = ptt * amount / sum_ratio
             if trade_type == "short":
                 amounts[currency] = - amounts[currency]
-            if currency not in self.all:
-                self.all[currency] = portfolio.Balance(currency, {})
+            self.all.setdefault(currency, portfolio.Balance(currency, {}))
         self.market.report.log_dispatch(amount, amounts, liquidity, repartition)
         return amounts
 
@@ -214,6 +221,10 @@ class TradeStore:
         self.market = market
         self.all = []
 
+    @property
+    def pending(self):
+        return list(filter(lambda t: not t.is_fullfiled, self.all))
+
     def compute_trades(self, values_in_base, new_repartition, only=None):
         computed_trades = []
         base_currency = sum(values_in_base.values()).currency
@@ -248,7 +259,7 @@ class TradeStore:
 
     def prepare_orders(self, only=None, compute_value="default"):
         orders = []
-        for trade in self.all:
+        for trade in self.pending:
             if only is None or trade.action == only:
                 orders.append(trade.prepare_order(compute_value=compute_value))
         self.market.report.log_orders(orders, only, compute_value)
diff --git a/test.py b/test.py
index 778fc14..141c9e0 100644
--- a/test.py
+++ b/test.py
@@ -35,10 +35,6 @@ class WebMockTestCase(unittest.TestCase):
                 mock.patch.multiple(portfolio.Portfolio, last_date=None, data=None, liquidities={}),
                 mock.patch.multiple(portfolio.Computation,
                     computations=portfolio.Computation.computations),
-                mock.patch.multiple(market.Market,
-                    fees_cache={},
-                    ticker_cache={},
-                    ticker_cache_timestamp=self.time.time()),
                 ]
         for patcher in self.patchers:
             patcher.start()
@@ -472,8 +468,9 @@ class BalanceTest(WebMockTestCase):
             "exchange_free": "0.35",
             "exchange_used": "0.30",
             "margin_total": "-10",
-            "margin_borrowed": "-10",
-            "margin_free": "0",
+            "margin_borrowed": "10",
+            "margin_available": "0",
+            "margin_in_position": "0",
             "margin_position_type": "short",
             "margin_borrowed_base_currency": "USDT",
             "margin_liquidation_price": "1.20",
@@ -489,11 +486,11 @@ class BalanceTest(WebMockTestCase):
         self.assertEqual("BTC", balance.exchange_total.currency)
 
         self.assertEqual(portfolio.D("-10"), balance.margin_total.value)
-        self.assertEqual(portfolio.D("-10"), balance.margin_borrowed.value)
-        self.assertEqual(portfolio.D("0"), balance.margin_free.value)
+        self.assertEqual(portfolio.D("10"), balance.margin_borrowed.value)
+        self.assertEqual(portfolio.D("0"), balance.margin_available.value)
         self.assertEqual("BTC", balance.margin_total.currency)
         self.assertEqual("BTC", balance.margin_borrowed.currency)
-        self.assertEqual("BTC", balance.margin_free.currency)
+        self.assertEqual("BTC", balance.margin_available.currency)
 
         self.assertEqual("BTC", balance.currency)
 
@@ -511,10 +508,10 @@ class BalanceTest(WebMockTestCase):
         self.assertEqual("Balance(BTX Exch: [❌1.00000000 BTX])", repr(balance))
 
         balance = portfolio.Balance("BTX", { "margin_total": 3,
-            "margin_borrowed": 1, "margin_free": 2 })
-        self.assertEqual("Balance(BTX Margin: [✔2.00000000 BTX + borrowed 1.00000000 BTX = 3.00000000 BTX])", repr(balance))
+            "margin_in_position": 1, "margin_available": 2 })
+        self.assertEqual("Balance(BTX Margin: [✔2.00000000 BTX + ❌1.00000000 BTX = 3.00000000 BTX])", repr(balance))
 
-        balance = portfolio.Balance("BTX", { "margin_total": 2, "margin_free": 2 })
+        balance = portfolio.Balance("BTX", { "margin_total": 2, "margin_available": 2 })
         self.assertEqual("Balance(BTX Margin: [✔2.00000000 BTX])", repr(balance))
 
         balance = portfolio.Balance("BTX", { "margin_total": -3,
@@ -524,8 +521,8 @@ class BalanceTest(WebMockTestCase):
         self.assertEqual("Balance(BTX Margin: [-3.00000000 BTX @@ 0.10000000 BTC/0.00200000 BTC])", repr(balance))
 
         balance = portfolio.Balance("BTX", { "margin_total": 1,
-            "margin_borrowed": 1, "exchange_free": 2, "exchange_total": 2})
-        self.assertEqual("Balance(BTX Exch: [✔2.00000000 BTX] Margin: [borrowed 1.00000000 BTX] Total: [0.00000000 BTX])", repr(balance))
+            "margin_in_position": 1, "exchange_free": 2, "exchange_total": 2})
+        self.assertEqual("Balance(BTX Exch: [✔2.00000000 BTX] Margin: [❌1.00000000 BTX] Total: [0.00000000 BTX])", repr(balance))
 
     def test_as_json(self):
         balance = portfolio.Balance("BTX", { "exchange_free": 2, "exchange_total": 2 })
@@ -536,7 +533,7 @@ class BalanceTest(WebMockTestCase):
         self.assertEqual(D(2), as_json["exchange_free"])
         self.assertEqual(D(0), as_json["exchange_used"])
         self.assertEqual(D(0), as_json["margin_total"])
-        self.assertEqual(D(0), as_json["margin_free"])
+        self.assertEqual(D(0), as_json["margin_available"])
         self.assertEqual(D(0), as_json["margin_borrowed"])
 
 @unittest.skipUnless("unit" in limits, "Unit skipped")
@@ -583,77 +580,69 @@ class MarketTest(WebMockTestCase):
         m = market.Market.from_config({"key": "key", "secred": "secret"}, debug=True)
         self.assertEqual(True, m.debug)
 
-    def test_get_ticker(self):
-        m = market.Market(self.ccxt)
-        self.ccxt.fetch_ticker.side_effect = [
-                { "bid": 1, "ask": 3 },
-                market.ExchangeError("foo"),
-                { "bid": 10, "ask": 40 },
-                market.ExchangeError("foo"),
-                market.ExchangeError("foo"),
+    def test_get_tickers(self):
+        self.ccxt.fetch_tickers.side_effect = [
+                "tickers",
+                market.NotSupported
                 ]
 
-        ticker = m.get_ticker("ETH", "ETC")
-        self.ccxt.fetch_ticker.assert_called_with("ETH/ETC")
-        self.assertEqual(1, ticker["bid"])
-        self.assertEqual(3, ticker["ask"])
-        self.assertEqual(2, ticker["average"])
-        self.assertFalse(ticker["inverted"])
-
-        ticker = m.get_ticker("ETH", "XVG")
-        self.assertEqual(0.0625, ticker["average"])
-        self.assertTrue(ticker["inverted"])
-        self.assertIn("original", ticker)
-        self.assertEqual(10, ticker["original"]["bid"])
-
-        ticker = m.get_ticker("XVG", "XMR")
-        self.assertIsNone(ticker)
-
-        self.ccxt.fetch_ticker.assert_has_calls([
-            mock.call("ETH/ETC"),
-            mock.call("ETH/XVG"),
-            mock.call("XVG/ETH"),
-            mock.call("XVG/XMR"),
-            mock.call("XMR/XVG"),
-            ])
+        m = market.Market(self.ccxt)
+        self.assertEqual("tickers", m.get_tickers())
+        self.assertEqual("tickers", m.get_tickers())
+        self.ccxt.fetch_tickers.assert_called_once()
 
-        self.ccxt = mock.Mock(spec=market.ccxt.poloniexE)
-        m1b = market.Market(self.ccxt)
-        m1b.get_ticker("ETH", "ETC")
-        self.ccxt.fetch_ticker.assert_not_called()
-
-        self.ccxt = mock.Mock(spec=market.ccxt.poloniex)
-        m2 = market.Market(self.ccxt)
-        self.ccxt.fetch_ticker.side_effect = [
-                { "bid": 1, "ask": 3 },
-                { "bid": 1.2, "ask": 3.5 },
-                ]
-        ticker1 = m2.get_ticker("ETH", "ETC")
-        ticker2 = m2.get_ticker("ETH", "ETC")
-        ticker3 = m2.get_ticker("ETC", "ETH")
-        self.ccxt.fetch_ticker.assert_called_once_with("ETH/ETC")
-        self.assertEqual(1, ticker1["bid"])
-        self.assertDictEqual(ticker1, ticker2)
-        self.assertDictEqual(ticker1, ticker3["original"])
-
-        ticker4 = m2.get_ticker("ETH", "ETC", refresh=True)
-        ticker5 = m2.get_ticker("ETH", "ETC")
-        self.assertEqual(1.2, ticker4["bid"])
-        self.assertDictEqual(ticker4, ticker5)
-
-        self.ccxt = mock.Mock(spec=market.ccxt.binance)
-        m3 = market.Market(self.ccxt)
-        self.ccxt.fetch_ticker.side_effect = [
-                { "bid": 1, "ask": 3 },
-                { "bid": 1.2, "ask": 3.5 },
-                ]
-        ticker6 = m3.get_ticker("ETH", "ETC")
-        m3.ticker_cache_timestamp -= 4
-        ticker7 = m3.get_ticker("ETH", "ETC")
-        m3.ticker_cache_timestamp -= 2
-        ticker8 = m3.get_ticker("ETH", "ETC")
-        self.assertDictEqual(ticker6, ticker7)
-        self.assertEqual(1.2, ticker8["bid"])
+        self.assertIsNone(m.get_tickers(refresh=self.time.time()))
+
+    def test_get_ticker(self):
+        with self.subTest(get_tickers=True):
+            self.ccxt.fetch_tickers.return_value = {
+                    "ETH/ETC": { "bid": 1, "ask": 3 },
+                    "XVG/ETH": { "bid": 10, "ask": 40 },
+                    }
+            m = market.Market(self.ccxt)
+
+            ticker = m.get_ticker("ETH", "ETC")
+            self.assertEqual(1, ticker["bid"])
+            self.assertEqual(3, ticker["ask"])
+            self.assertEqual(2, ticker["average"])
+            self.assertFalse(ticker["inverted"])
+
+            ticker = m.get_ticker("ETH", "XVG")
+            self.assertEqual(0.0625, ticker["average"])
+            self.assertTrue(ticker["inverted"])
+            self.assertIn("original", ticker)
+            self.assertEqual(10, ticker["original"]["bid"])
+
+            ticker = m.get_ticker("XVG", "XMR")
+            self.assertIsNone(ticker)
+
+        with self.subTest(get_tickers=False):
+            self.ccxt.fetch_tickers.return_value = None
+            self.ccxt.fetch_ticker.side_effect = [
+                    { "bid": 1, "ask": 3 },
+                    market.ExchangeError("foo"),
+                    { "bid": 10, "ask": 40 },
+                    market.ExchangeError("foo"),
+                    market.ExchangeError("foo"),
+                    ]
+
+            m = market.Market(self.ccxt)
+
+            ticker = m.get_ticker("ETH", "ETC")
+            self.ccxt.fetch_ticker.assert_called_with("ETH/ETC")
+            self.assertEqual(1, ticker["bid"])
+            self.assertEqual(3, ticker["ask"])
+            self.assertEqual(2, ticker["average"])
+            self.assertFalse(ticker["inverted"])
+
+            ticker = m.get_ticker("ETH", "XVG")
+            self.assertEqual(0.0625, ticker["average"])
+            self.assertTrue(ticker["inverted"])
+            self.assertIn("original", ticker)
+            self.assertEqual(10, ticker["original"]["bid"])
+
+            ticker = m.get_ticker("XVG", "XMR")
+            self.assertIsNone(ticker)
 
     def test_fetch_fees(self):
         m = market.Market(self.ccxt)
@@ -906,9 +895,9 @@ class MarketTest(WebMockTestCase):
                 trade3 = portfolio.Trade(value_from, value_to, "XVG", m)
 
                 m.trades.all = [trade1, trade2, trade3]
-                balance1 = portfolio.Balance("BTC", { "margin_free": "0" })
-                balance2 = portfolio.Balance("USDT", { "margin_free": "100" })
-                balance3 = portfolio.Balance("ETC", { "margin_free": "10" })
+                balance1 = portfolio.Balance("BTC", { "margin_in_position": "0", "margin_available": "0" })
+                balance2 = portfolio.Balance("USDT", { "margin_in_position": "100", "margin_available": "50" })
+                balance3 = portfolio.Balance("ETC", { "margin_in_position": "10", "margin_available": "15" })
                 m.balances.all = {"BTC": balance1, "USDT": balance2, "ETC": balance3}
 
                 m.move_balances()
@@ -921,8 +910,8 @@ class MarketTest(WebMockTestCase):
                     self.assertEqual(3, m.report.log_debug_action.call_count)
                 else:
                     self.ccxt.transfer_balance.assert_any_call("BTC", 3, "exchange", "margin")
-                    self.ccxt.transfer_balance.assert_any_call("USDT", 50, "margin", "exchange")
-                    self.ccxt.transfer_balance.assert_any_call("ETC", 10, "margin", "exchange")
+                    self.ccxt.transfer_balance.assert_any_call("USDT", 100, "exchange", "margin")
+                    self.ccxt.transfer_balance.assert_any_call("ETC", 5, "margin", "exchange")
   
 @unittest.skipUnless("unit" in limits, "Unit skipped")
 class TradeStoreTest(WebMockTestCase):
@@ -1001,16 +990,24 @@ class TradeStoreTest(WebMockTestCase):
 
         trade_mock1 = mock.Mock()
         trade_mock2 = mock.Mock()
+        trade_mock3 = mock.Mock()
 
         trade_mock1.prepare_order.return_value = 1
         trade_mock2.prepare_order.return_value = 2
+        trade_mock3.prepare_order.return_value = 3
+
+        trade_mock1.is_fullfiled = False
+        trade_mock2.is_fullfiled = False
+        trade_mock3.is_fullfiled = True
 
         trade_store.all.append(trade_mock1)
         trade_store.all.append(trade_mock2)
+        trade_store.all.append(trade_mock3)
 
         trade_store.prepare_orders()
         trade_mock1.prepare_order.assert_called_with(compute_value="default")
         trade_mock2.prepare_order.assert_called_with(compute_value="default")
+        trade_mock3.prepare_order.assert_not_called()
         self.m.report.log_orders.assert_called_once_with([1, 2], None, "default")
 
         self.m.report.log_orders.reset_mock()
@@ -1108,6 +1105,21 @@ class TradeStoreTest(WebMockTestCase):
             order_mock2.get_status.assert_called()
             order_mock3.get_status.assert_called()
 
+    def test_pending(self):
+        trade_mock1 = mock.Mock()
+        trade_mock1.is_fullfiled = False
+        trade_mock2 = mock.Mock()
+        trade_mock2.is_fullfiled = False
+        trade_mock3 = mock.Mock()
+        trade_mock3.is_fullfiled = True
+
+        trade_store = market.TradeStore(self.m)
+
+        trade_store.all.append(trade_mock1)
+        trade_store.all.append(trade_mock2)
+        trade_store.all.append(trade_mock3)
+
+        self.assertEqual([trade_mock1, trade_mock2], trade_store.pending)
 
 @unittest.skipUnless("unit" in limits, "Unit skipped")
 class BalanceStoreTest(WebMockTestCase):
@@ -1301,13 +1313,16 @@ class TradeTest(WebMockTestCase):
         self.assertEqual(self.m, trade.market)
 
         with self.assertRaises(AssertionError):
-            portfolio.Trade(value_from, value_to, "ETC", self.m)
+            portfolio.Trade(value_from, -value_to, "ETH", self.m)
         with self.assertRaises(AssertionError):
-            value_from.linked_to = None
-            portfolio.Trade(value_from, value_to, "ETH", self.m)
+            portfolio.Trade(value_from, value_to, "ETC", self.m)
         with self.assertRaises(AssertionError):
             value_from.currency = "ETH"
             portfolio.Trade(value_from, value_to, "ETH", self.m)
+            value_from.currency = "BTC"
+        with self.assertRaises(AssertionError):
+            value_from2 = portfolio.Amount("BTC", "1.0")
+            portfolio.Trade(value_from2, value_to, "ETH", self.m)
 
         value_from = portfolio.Amount("BTC", 0)
         trade = portfolio.Trade(value_from, value_to, "ETH", self.m)
@@ -1374,6 +1389,28 @@ class TradeTest(WebMockTestCase):
 
         self.assertEqual("short", trade.trade_type)
 
+    def test_is_fullfiled(self):
+        value_from = portfolio.Amount("BTC", "0.5")
+        value_from.linked_to = portfolio.Amount("ETH", "10.0")
+        value_to = portfolio.Amount("BTC", "1.0")
+        trade = portfolio.Trade(value_from, value_to, "ETH", self.m)
+
+        order1 = mock.Mock()
+        order1.filled_amount.return_value = portfolio.Amount("BTC", "0.3")
+
+        order2 = mock.Mock()
+        order2.filled_amount.return_value = portfolio.Amount("BTC", "0.01")
+        trade.orders.append(order1)
+        trade.orders.append(order2)
+
+        self.assertFalse(trade.is_fullfiled)
+
+        order3 = mock.Mock()
+        order3.filled_amount.return_value = portfolio.Amount("BTC", "0.19")
+        trade.orders.append(order3)
+
+        self.assertTrue(trade.is_fullfiled)
+
     def test_filled_amount(self):
         value_from = portfolio.Amount("BTC", "0.5")
         value_from.linked_to = portfolio.Amount("ETH", "10.0")
@@ -1588,7 +1625,7 @@ class TradeTest(WebMockTestCase):
         self.assertEqual(2, self.m.report.log_order.call_count)
         calls = [
                 mock.call(order_mock, 2, update="adjusting",
-                    compute_value='lambda x, y: (x[y] + x["average"]) / 2',
+                    compute_value=mock.ANY,
                     new_order=new_order_mock),
                 mock.call(order_mock, 2, new_order=new_order_mock),
                 ]
@@ -1607,7 +1644,7 @@ class TradeTest(WebMockTestCase):
         self.m.report.log_order.assert_called()
         calls = [
                 mock.call(order_mock, 5, update="adjusting",
-                    compute_value='lambda x, y: (x[y]*2 + x["average"]) / 3',
+                    compute_value=mock.ANY,
                     new_order=new_order_mock),
                 mock.call(order_mock, 5, new_order=new_order_mock),
                 ]
@@ -1831,7 +1868,7 @@ class OrderTest(WebMockTestCase):
 
         order.cancel()
         self.m.ccxt.cancel_order.assert_called_with(42)
-        fetch.assert_called_once_with(force=True)
+        fetch.assert_called_once_with()
         self.m.report.log_debug_action.assert_not_called()
 
     def test_dust_amount_remaining(self):
@@ -1850,11 +1887,9 @@ class OrderTest(WebMockTestCase):
                 D("0.1"), "BTC", "long", self.m, "trade")
 
         self.assertEqual(9, order.remaining_amount().value)
-        order.fetch.assert_not_called()
 
         order.status = "open"
         self.assertEqual(9, order.remaining_amount().value)
-        fetch.assert_called_once()
 
     @mock.patch.object(portfolio.Order, "fetch")
     def test_filled_amount(self, fetch):
@@ -1957,61 +1992,38 @@ class OrderTest(WebMockTestCase):
 
     @mock.patch.object(portfolio.Order, "fetch_mouvements")
     def test_fetch(self, fetch_mouvements):
-        time = self.time.time()
-        with mock.patch.object(portfolio.time, "time") as time_mock:
-            order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
-                    D("0.1"), "BTC", "long", self.m, "trade")
-            order.id = 45
-            with self.subTest(debug=True):
-                self.m.debug = True
-                order.fetch()
-                time_mock.assert_not_called()
-                self.m.report.log_debug_action.assert_called_once()
-                self.m.report.log_debug_action.reset_mock()
-                order.fetch(force=True)
-                time_mock.assert_not_called()
-                self.m.ccxt.fetch_order.assert_not_called()
-                fetch_mouvements.assert_not_called()
-                self.m.report.log_debug_action.assert_called_once()
-                self.m.report.log_debug_action.reset_mock()
-            self.assertIsNone(order.fetch_cache_timestamp)
-
-            with self.subTest(debug=False):
-                self.m.debug = False
-                time_mock.return_value = time
-                self.m.ccxt.fetch_order.return_value = {
-                        "status": "foo",
-                        "datetime": "timestamp"
-                        }
-                order.fetch()
-
-                self.m.ccxt.fetch_order.assert_called_once_with(45, symbol="ETH")
-                fetch_mouvements.assert_called_once()
-                self.assertEqual("foo", order.status)
-                self.assertEqual("timestamp", order.timestamp)
-                self.assertEqual(time, order.fetch_cache_timestamp)
-                self.assertEqual(1, len(order.results))
-
-                self.m.ccxt.fetch_order.reset_mock()
-                fetch_mouvements.reset_mock()
-
-                time_mock.return_value = time + 8
-                order.fetch()
-                self.m.ccxt.fetch_order.assert_not_called()
-                fetch_mouvements.assert_not_called()
+        order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
+                D("0.1"), "BTC", "long", self.m, "trade")
+        order.id = 45
+        with self.subTest(debug=True):
+            self.m.debug = True
+            order.fetch()
+            self.m.report.log_debug_action.assert_called_once()
+            self.m.report.log_debug_action.reset_mock()
+            self.m.ccxt.fetch_order.assert_not_called()
+            fetch_mouvements.assert_not_called()
 
-                order.fetch(force=True)
-                self.m.ccxt.fetch_order.assert_called_once_with(45, symbol="ETH")
-                fetch_mouvements.assert_called_once()
+        with self.subTest(debug=False):
+            self.m.debug = False
+            self.m.ccxt.fetch_order.return_value = {
+                    "status": "foo",
+                    "datetime": "timestamp"
+                    }
+            order.fetch()
 
-                self.m.ccxt.fetch_order.reset_mock()
-                fetch_mouvements.reset_mock()
+            self.m.ccxt.fetch_order.assert_called_once_with(45, symbol="ETH")
+            fetch_mouvements.assert_called_once()
+            self.assertEqual("foo", order.status)
+            self.assertEqual("timestamp", order.timestamp)
+            self.assertEqual(1, len(order.results))
+            self.m.report.log_debug_action.assert_not_called()
 
-                time_mock.return_value = time + 19
+            with self.subTest(missing_order=True):
+                self.m.ccxt.fetch_order.side_effect = [
+                        portfolio.OrderNotCached,
+                        ]
                 order.fetch()
-                self.m.ccxt.fetch_order.assert_called_once_with(45, symbol="ETH")
-                fetch_mouvements.assert_called_once()
-                self.m.report.log_debug_action.assert_not_called()
+                self.assertEqual("closed_unknown", order.status)
 
     @mock.patch.object(portfolio.Order, "fetch")
     @mock.patch.object(portfolio.Order, "mark_finished_order")
@@ -2315,6 +2327,25 @@ class ReportStoreTest(WebMockTestCase):
             'total': D('10.3')
             })
 
+        add_log.reset_mock()
+        compute_value = lambda x: x["bid"]
+        report_store.log_tickers(amounts, "BTC", compute_value, "total")
+        add_log.assert_called_once_with({
+            'type': 'tickers',
+            'compute_value': 'compute_value = lambda x: x["bid"]',
+            'balance_type': 'total',
+            'currency': 'BTC',
+            'balances': {
+                'BTC': D('10'),
+                'ETH': D('0.3')
+                },
+            'rates': {
+                'BTC': None,
+                'ETH': D('0.1')
+                },
+            'total': D('10.3')
+            })
+
     @mock.patch.object(market.ReportStore, "print_log")
     @mock.patch.object(market.ReportStore, "add_log")
     def test_log_dispatch(self, add_log, print_log):
@@ -2393,6 +2424,20 @@ class ReportStoreTest(WebMockTestCase):
             'orders': ['order1', 'order2']
             })
 
+        add_log.reset_mock()
+        def compute_value(x, y):
+            return x[y]
+        report_store.log_orders(orders, tick="tick",
+                only="only", compute_value=compute_value)
+        add_log.assert_called_with({
+            'type': 'orders',
+            'only': 'only',
+            'compute_value': 'def compute_value(x, y):\n            return x[y]',
+            'tick': 'tick',
+            'orders': ['order1', 'order2']
+            })
+
+
     @mock.patch.object(market.ReportStore, "print_log")
     @mock.patch.object(market.ReportStore, "add_log")
     def test_log_order(self, add_log, print_log):
@@ -2436,16 +2481,17 @@ class ReportStoreTest(WebMockTestCase):
         add_log.reset_mock()
         print_log.reset_mock()
         with self.subTest(update="adjusting"):
+            compute_value = lambda x: (x["bid"] + x["ask"]*2)/3
             report_store.log_order(order_mock, 3,
                     update="adjusting", new_order=new_order_mock,
-                    compute_value="default")
+                    compute_value=compute_value)
             print_log.assert_called_once_with("[Order] Order Mock, tick 3, cancelling and adjusting to New order Mock")
             add_log.assert_called_once_with({
                 'type': 'order',
                 'tick': 3,
                 'update': 'adjusting',
                 'order': 'order',
-                'compute_value': "default",
+                'compute_value': 'compute_value = lambda x: (x["bid"] + x["ask"]*2)/3',
                 'new_order': 'new_order'
                 })