Add acceptance tests

[perso/Immae/Projets/Cryptomonnaies/Cryptoportfolio/Trader.git] / portfolio.py

import datetime
from retry import retry
from decimal import Decimal as D, ROUND_DOWN
from ccxt import ExchangeError, InsufficientFunds, ExchangeNotAvailable, InvalidOrder, OrderNotCached, OrderNotFound, RequestTimeout, InvalidNonce

class Computation:
    computations = {
            "default": lambda x, y: x[y],
            "average": lambda x, y: x["average"],
            "bid": lambda x, y: x["bid"],
            "ask": lambda x, y: x["ask"],
            }

    @classmethod
    def compute_value(cls, ticker, action, compute_value="default"):
        if action == "buy":
            action = "ask"
        if action == "sell":
            action = "bid"
        if isinstance(compute_value, str):
            compute_value = cls.computations[compute_value]
        return compute_value(ticker, action)

class Amount:
    def __init__(self, currency, value, linked_to=None, ticker=None, rate=None):
        self.currency = currency
        self.value = D(value)
        self.linked_to = linked_to
        self.ticker = ticker
        self.rate = rate

    def in_currency(self, other_currency, market, rate=None, action=None, compute_value="average"):
        if other_currency == self.currency:
            return self
        if rate is not None:
            return Amount(
                    other_currency,
                    self.value * rate,
                    linked_to=self,
                    rate=rate)
        asset_ticker = market.get_ticker(self.currency, other_currency)
        if asset_ticker is not None:
            rate = Computation.compute_value(asset_ticker, action, compute_value=compute_value)
            return Amount(
                    other_currency,
                    self.value * rate,
                    linked_to=self,
                    ticker=asset_ticker,
                    rate=rate)
        else:
            raise Exception("This asset is not available in the chosen market")

    def as_json(self):
        return {
                "currency": self.currency,
                "value": round(self).value.normalize(),
                }

    def __round__(self, n=8):
        return Amount(self.currency, self.value.quantize(D(1)/D(10**n), rounding=ROUND_DOWN))

    def __abs__(self):
        return Amount(self.currency, abs(self.value))

    def __add__(self, other):
        if other == 0:
            return self
        if other.currency != self.currency and other.value * self.value != 0:
            raise Exception("Summing amounts must be done with same currencies")
        return Amount(self.currency, self.value + other.value)

    def __radd__(self, other):
        if other == 0:
            return self
        else:
            return self.__add__(other)

    def __sub__(self, other):
        if other == 0:
            return self
        if other.currency != self.currency and other.value * self.value != 0:
            raise Exception("Summing amounts must be done with same currencies")
        return Amount(self.currency, self.value - other.value)

    def __rsub__(self, other):
        if other == 0:
            return -self
        else:
            return -self.__sub__(other)

    def __mul__(self, value):
        if not isinstance(value, (int, float, D)):
            raise TypeError("Amount may only be multiplied by numbers")
        return Amount(self.currency, self.value * value)

    def __rmul__(self, value):
        return self.__mul__(value)

    def __floordiv__(self, value):
        if not isinstance(value, (int, float, D)):
            raise TypeError("Amount may only be divided by numbers")
        return Amount(self.currency, self.value / value)

    def __truediv__(self, value):
        return self.__floordiv__(value)

    def __lt__(self, other):
        if other == 0:
            return self.value < 0
        if self.currency != other.currency:
            raise Exception("Comparing amounts must be done with same currencies")
        return self.value < other.value

    def __le__(self, other):
        return self == other or self < other

    def __gt__(self, other):
        return not self <= other

    def __ge__(self, other):
        return not self < other

    def __eq__(self, other):
        if other == 0:
            return self.value == 0
        if self.currency != other.currency:
            raise Exception("Comparing amounts must be done with same currencies")
        return self.value == other.value

    def __ne__(self, other):
        return not self == other

    def __neg__(self):
        return Amount(self.currency, - self.value)

    def __str__(self):
        if self.linked_to is None:
            return "{:.8f} {}".format(self.value, self.currency)
        else:
            return "{:.8f} {} [{}]".format(self.value, self.currency, self.linked_to)

    def __repr__(self):
        if self.linked_to is None:
            return "Amount({:.8f} {})".format(self.value, self.currency)
        else:
            return "Amount({:.8f} {} -> {})".format(self.value, self.currency, repr(self.linked_to))

class Balance:
    base_keys = ["total", "exchange_total", "exchange_used",
            "exchange_free", "margin_total", "margin_in_position",
            "margin_available", "margin_borrowed", "margin_pending_gain"]

    def __init__(self, currency, hash_):
        self.currency = currency
        for key in self.base_keys:
            setattr(self, key, Amount(currency, hash_.get(key, 0)))

        self.margin_position_type = hash_.get("margin_position_type")

        if hash_.get("margin_borrowed_base_currency") is not None:
            base_currency = hash_["margin_borrowed_base_currency"]
            for key in [
                    "margin_liquidation_price",
                    "margin_lending_fees",
                    "margin_pending_base_gain",
                    "margin_borrowed_base_price"
                    ]:
                setattr(self, key, Amount(base_currency, hash_.get(key, 0)))

    def as_json(self):
        return dict(map(lambda x: (x, getattr(self, x).as_json()["value"]), self.base_keys))

    def __repr__(self):
        if self.exchange_total > 0:
            if self.exchange_free > 0 and self.exchange_used > 0:
                exchange = " Exch: [✔{} + ❌{} = {}]".format(str(self.exchange_free), str(self.exchange_used), str(self.exchange_total))
            elif self.exchange_free > 0:
                exchange = " Exch: [✔{}]".format(str(self.exchange_free))
            else:
                exchange = " Exch: [❌{}]".format(str(self.exchange_used))
        else:
            exchange = ""

        if self.margin_total > 0:
            if self.margin_available != 0 and self.margin_in_position != 0:
                margin = " Margin: [✔{} + ❌{} = {}]".format(str(self.margin_available), str(self.margin_in_position), str(self.margin_total))
            elif self.margin_available != 0:
                margin = " Margin: [✔{}]".format(str(self.margin_available))
            else:
                margin = " Margin: [❌{}]".format(str(self.margin_in_position))
        elif self.margin_total < 0:
            margin = " Margin: [{} @@ {}/{}]".format(str(self.margin_total),
                    str(self.margin_borrowed_base_price),
                    str(self.margin_lending_fees))
        else:
            margin = ""

        if self.margin_total != 0 and self.exchange_total != 0:
            total = " Total: [{}]".format(str(self.total))
        else:
            total = ""

        return "Balance({}".format(self.currency) + "".join([exchange, margin, total]) + ")"

class Trade:
    def __init__(self, value_from, value_to, currency, market):
        # We have value_from of currency, and want to finish with value_to of
        # that currency. value_* may not be in currency's terms
        self.currency = currency
        self.value_from = value_from
        self.value_to = value_to
        self.orders = []
        self.market = market
        self.closed = False
        self.inverted = None
        assert self.value_from.value * self.value_to.value >= 0
        assert self.value_from.currency == self.value_to.currency
        if self.value_from != 0:
            assert self.value_from.linked_to is not None and self.value_from.linked_to.currency == self.currency
        elif self.value_from.linked_to is None:
            self.value_from.linked_to = Amount(self.currency, 0)
        self.base_currency = self.value_from.currency

    @property
    def delta(self):
        return self.value_to - self.value_from

    @property
    def action(self):
        if self.value_from == self.value_to:
            return None
        if self.base_currency == self.currency:
            return None

        if abs(self.value_from) < abs(self.value_to):
            return "acquire"
        else:
            return "dispose"

    def order_action(self):
        if (self.value_from < self.value_to) != self.inverted:
            return "buy"
        else:
            return "sell"

    @property
    def trade_type(self):
        if self.value_from + self.value_to < 0:
            return "short"
        else:
            return "long"

    @property
    def pending(self):
        return not (self.is_fullfiled or self.closed)

    def close(self):
        for order in self.orders:
            order.cancel()
        self.closed = True

    @property
    def is_fullfiled(self):
        return abs(self.filled_amount(in_base_currency=(not self.inverted))) >= abs(self.delta)

    def filled_amount(self, in_base_currency=False):
        filled_amount = 0
        for order in self.orders:
            filled_amount += order.filled_amount(in_base_currency=in_base_currency)
        return filled_amount

    tick_actions = {
            0: ["waiting", None],
            1: ["waiting", None],
            2: ["adjusting", lambda x, y: (x[y] + x["average"]) / 2],
            3: ["waiting", None],
            4: ["waiting", None],
            5: ["adjusting", lambda x, y: (x[y]*2 + x["average"]) / 3],
            6: ["waiting", None],
            7: ["market_fallback", "default"],
            }

    def tick_actions_recreate(self, tick, default="average"):
        return ([default] + \
                [ y[1] for x, y in self.tick_actions.items() if x <= tick and y[1] is not None ])[-1]

    def update_order(self, order, tick):
        if tick in self.tick_actions:
            update, compute_value = self.tick_actions[tick]
        elif tick % 3 == 1:
            update = "market_adjust"
            compute_value = "default"
        else:
            update = "waiting"
            compute_value = None

        if compute_value is not None:
            order.cancel()
            new_order = self.prepare_order(compute_value=compute_value)
        else:
            new_order = None

        self.market.report.log_order(order, tick, update=update,
                compute_value=compute_value, new_order=new_order)

        if new_order is not None:
            new_order.run()
            self.market.report.log_order(order, tick, new_order=new_order)

    def prepare_order(self, close_if_possible=None, compute_value="default"):
        if self.action is None:
            return None
        ticker = self.market.get_ticker(self.currency, self.base_currency)
        self.inverted = ticker["inverted"]
        if self.inverted:
            ticker = ticker["original"]
        rate = Computation.compute_value(ticker, self.order_action(), compute_value=compute_value)

        # FIXME: Dust amount should be removed from there if they werent
        # honored in other sales
        delta_in_base = abs(self.delta)
        # 9 BTC's worth of move (10 - 1 or 1 - 10 depending on case)

        if not self.inverted:
            base_currency = self.base_currency
            # BTC
            if self.action == "dispose":
                filled = self.filled_amount(in_base_currency=False)
                delta = delta_in_base.in_currency(self.currency, self.market, rate=1/self.value_from.rate)
                # I have 10 BTC worth of FOO, and I want to sell 9 BTC
                # worth of it, computed first with rate 10 FOO = 1 BTC.
                # -> I "sell" "90" FOO at proposed rate "rate".

                delta = delta - filled
                # I already sold 60 FOO, 30 left
            else:
                filled = self.filled_amount(in_base_currency=True)
                delta = (delta_in_base - filled).in_currency(self.currency, self.market, rate=1/rate)
                # I want to buy 9 BTC worth of FOO, computed with rate
                # 10 FOO = 1 BTC
                # -> I "buy" "9 / rate" FOO at proposed rate "rate"

                # I already bought 3 / rate FOO, 6 / rate left
        else:
            base_currency = self.currency
            # FOO
            if self.action == "dispose":
                filled = self.filled_amount(in_base_currency=True)
                # Base is FOO

                delta = (delta_in_base.in_currency(self.currency, self.market, rate=1/self.value_from.rate)
                        - filled).in_currency(self.base_currency, self.market, rate=1/rate)
                # I have 10 BTC worth of FOO, and I want to sell 9 BTC worth of it
                # computed at rate 1 Foo = 0.01 BTC
                # Computation says I should sell it at 125 FOO / BTC
                # -> delta_in_base = 9 BTC
                # -> delta = (9 * 1/0.01 FOO) * 1/125 = 7.2 BTC
                # Action: "buy" "7.2 BTC" at rate "125" "FOO" on market

                # I already bought 300/125 BTC, only 600/125 left
            else:
                filled = self.filled_amount(in_base_currency=False)
                # Base is FOO

                delta = delta_in_base
                # I have 1 BTC worth of FOO, and I want to buy 9 BTC worth of it
                # At rate 100 Foo / BTC
                # Computation says I should buy it at 125 FOO / BTC
                # -> delta_in_base = 9 BTC
                # Action: "sell" "9 BTC" at rate "125" "FOO" on market

                delta = delta - filled
                # I already sold 4 BTC, only 5 left

        if close_if_possible is None:
            close_if_possible = (self.value_to == 0)

        if delta <= 0:
            self.market.report.log_error("prepare_order", message="Less to do than already filled: {}".format(delta))
            return None

        order = Order(self.order_action(),
            delta, rate, base_currency, self.trade_type,
            self.market, self, close_if_possible=close_if_possible)
        self.orders.append(order)
        return order

    def as_json(self):
        return {
                "action": self.action,
                "from": self.value_from.as_json()["value"],
                "to": self.value_to.as_json()["value"],
                "currency": self.currency,
                "base_currency": self.base_currency,
                }

    def __repr__(self):
        if self.closed and not self.is_fullfiled:
            closed = " ❌"
        elif self.is_fullfiled:
            closed = " ✔"
        else:
            closed = ""

        return "Trade({} -> {} in {}, {}{})".format(
                self.value_from,
                self.value_to,
                self.currency,
                self.action,
                closed)

    def print_with_order(self, ind=""):
        self.market.report.print_log("{}{}".format(ind, self))
        for order in self.orders:
            self.market.report.print_log("{}\t{}".format(ind, order))
            for mouvement in order.mouvements:
                self.market.report.print_log("{}\t\t{}".format(ind, mouvement))

class RetryException(Exception):
    pass

class Order:
    def __init__(self, action, amount, rate, base_currency, trade_type, market,
            trade, close_if_possible=False):
        self.action = action
        self.amount = amount
        self.rate = rate
        self.base_currency = base_currency
        self.market = market
        self.trade_type = trade_type
        self.results = []
        self.mouvements = []
        self.status = "pending"
        self.trade = trade
        self.close_if_possible = close_if_possible
        self.id = None
        self.tries = 0
        self.start_date = None

    def as_json(self):
        return {
                "action": self.action,
                "trade_type": self.trade_type,
                "amount": self.amount.as_json()["value"],
                "currency": self.amount.as_json()["currency"],
                "base_currency": self.base_currency,
                "rate": self.rate,
                "status": self.status,
                "close_if_possible": self.close_if_possible,
                "id": self.id,
                "mouvements": list(map(lambda x: x.as_json(), self.mouvements))
                }

    def __repr__(self):
        return "Order({} {} {} at {} {} [{}]{})".format(
                self.action,
                self.trade_type,
                self.amount,
                self.rate,
                self.base_currency,
                self.status,
                " ✂" if self.close_if_possible else "",
                )

    @property
    def account(self):
        if self.trade_type == "long":
            return "exchange"
        else:
            return "margin"

    @property
    def open(self):
        return self.status == "open"

    @property
    def pending(self):
        return self.status == "pending"

    @property
    def finished(self):
        return self.status.startswith("closed") or self.status == "canceled" or self.status == "error"

    @retry((InsufficientFunds, RetryException, InvalidNonce))
    def run(self):
        self.tries += 1
        symbol = "{}/{}".format(self.amount.currency, self.base_currency)
        amount = round(self.amount, self.market.ccxt.order_precision(symbol)).value

        action = "market.ccxt.create_order('{}', 'limit', '{}', {}, price={}, account={})".format(symbol, self.action, amount, self.rate, self.account)
        if self.market.debug:
            self.market.report.log_debug_action(action)
            self.results.append({"debug": True, "id": -1})
        else:
            self.start_date = datetime.datetime.now()
            try:
                self.results.append(self.market.ccxt.create_order(symbol, 'limit', self.action, amount, price=self.rate, account=self.account))
            except InvalidOrder:
                # Impossible to honor the order (dust amount)
                self.status = "closed"
                self.mark_finished_order()
                return
            except InvalidNonce as e:
                if self.tries < 5:
                    self.market.report.log_error(action, message="Retrying after invalid nonce", exception=e)
                    raise e
                else:
                    self.market.report.log_error(action, message="Giving up {} after invalid nonce".format(self), exception=e)
                    self.status = "error"
                    return
            except RequestTimeout as e:
                if not self.retrieve_order():
                    if self.tries < 5:
                        self.market.report.log_error(action, message="Retrying after timeout", exception=e)
                        # We make a specific call in case retrieve_order
                        # would raise itself
                        raise RetryException
                    else:
                        self.market.report.log_error(action, message="Giving up {} after timeouts".format(self), exception=e)
                        self.status = "error"
                        return
                else:
                    self.market.report.log_error(action, message="Timeout, found the order")
            except InsufficientFunds as e:
                if self.tries < 5:
                    self.market.report.log_error(action, message="Retrying with reduced amount", exception=e)
                    self.amount = self.amount * D("0.99")
                    raise e
                else:
                    self.market.report.log_error(action, message="Giving up {}".format(self), exception=e)
                    self.status = "error"
                    return
            except Exception as e:
                self.status = "error"
                self.market.report.log_error(action, exception=e)
                return
        self.id = self.results[0]["id"]
        self.status = "open"

    def get_status(self):
        if self.market.debug:
            self.market.report.log_debug_action("Getting {} status".format(self))
            return self.status
        # other states are "closed" and "canceled"
        if not self.finished:
            self.fetch()
        return self.status

    def mark_disappeared_order(self):
        if self.status.startswith("closed") and \
                len(self.mouvements) > 0 and \
                self.mouvements[-1].total_in_base == 0:
            self.status = "error_disappeared"

    def mark_finished_order(self):
        if self.status.startswith("closed") and self.market.debug:
            self.market.report.log_debug_action("Mark {} as finished".format(self))
            return
        if self.status.startswith("closed"):
            if self.trade_type == "short" and self.action == "buy" and self.close_if_possible:
                self.market.ccxt.close_margin_position(self.amount.currency, self.base_currency)

    def fetch(self):
        if self.market.debug:
            self.market.report.log_debug_action("Fetching {}".format(self))
            return
        try:
            result = self.market.ccxt.fetch_order(self.id)
            self.results.append(result)
            self.status = result["status"]
            # Time at which the order started
            self.timestamp = result["datetime"]
        except OrderNotCached:
            self.status = "closed_unknown"

        self.fetch_mouvements()

        self.mark_disappeared_order()

        self.mark_finished_order()
        # FIXME: consider open order with dust remaining as closed

    def dust_amount_remaining(self):
        return self.remaining_amount() < Amount(self.amount.currency, D("0.001"))

    def remaining_amount(self):
        return self.amount - self.filled_amount()

    def filled_amount(self, in_base_currency=False):
        if self.status == "open":
            self.fetch()
        filled_amount = 0
        for mouvement in self.mouvements:
            if in_base_currency:
                filled_amount += mouvement.total_in_base
            else:
                filled_amount += mouvement.total
        return filled_amount

    def fetch_mouvements(self):
        try:
            mouvements = self.market.ccxt.privatePostReturnOrderTrades({"orderNumber": self.id})
        except ExchangeError:
            mouvements = []
        self.mouvements = []

        for mouvement_hash in mouvements:
            self.mouvements.append(Mouvement(self.amount.currency,
                self.base_currency, mouvement_hash))
        self.mouvements.sort(key= lambda x: x.date)

    def cancel(self):
        if self.market.debug:
            self.market.report.log_debug_action("Mark {} as cancelled".format(self))
            self.status = "canceled"
            return
        if self.open and self.id is not None:
            try:
                self.market.ccxt.cancel_order(self.id)
            except OrderNotFound as e: # Closed inbetween
                self.market.report.log_error("cancel_order", message="Already cancelled order", exception=e)
            self.fetch()

    def retrieve_order(self):
        symbol = "{}/{}".format(self.amount.currency, self.base_currency)
        amount = round(self.amount, self.market.ccxt.order_precision(symbol)).value
        start_timestamp = self.start_date.timestamp() - 5

        similar_open_orders = self.market.ccxt.fetch_orders(symbol=symbol, since=start_timestamp)
        for order in similar_open_orders:
            if (order["info"]["margin"] == 1 and self.account == "exchange") or\
                    (order["info"]["margin"] != 1 and self.account == "margin"):
                i_m_tested = True # coverage bug ?!
                continue
            if order["info"]["side"] != self.action:
                continue
            amount_diff = round(
                    abs(D(order["info"]["startingAmount"]) - amount),
                    self.market.ccxt.order_precision(symbol))
            rate_diff = round(
                    abs(D(order["info"]["rate"]) - self.rate),
                    self.market.ccxt.order_precision(symbol))
            if amount_diff != 0 or rate_diff != 0:
                continue
            self.results.append({"id": order["id"]})
            return True

        similar_trades = self.market.ccxt.fetch_my_trades(symbol=symbol, since=start_timestamp)
        # FIXME: use set instead of sorted(list(...))
        for order_id in sorted(list(map(lambda x: x["order"], similar_trades))):
            trades = list(filter(lambda x: x["order"] == order_id, similar_trades))
            if any(x["timestamp"] < start_timestamp for x in trades):
                continue
            if any(x["side"] != self.action for x in trades):
                continue
            if any(x["info"]["category"] == "exchange" and self.account == "margin" for x in trades) or\
                    any(x["info"]["category"] == "marginTrade" and self.account == "exchange" for x in trades):
                continue
            trade_sum = sum(D(x["info"]["amount"]) for x in trades)
            amount_diff = round(abs(trade_sum - amount),
                    self.market.ccxt.order_precision(symbol))
            if amount_diff != 0:
                continue
            if (self.action == "sell" and any(D(x["info"]["rate"]) < self.rate for x in trades)) or\
                    (self.action == "buy" and any(D(x["info"]["rate"]) > self.rate for x in trades)):
                continue
            self.results.append({"id": order_id})
            return True

        return False

class Mouvement:
    def __init__(self, currency, base_currency, hash_):
        self.currency = currency
        self.base_currency = base_currency
        self.id = hash_.get("tradeID")
        self.action = hash_.get("type")
        self.fee_rate = D(hash_.get("fee", -1))
        try:
            self.date = datetime.datetime.strptime(hash_.get("date", ""), '%Y-%m-%d %H:%M:%S')
        except ValueError:
            self.date = None
        self.rate = D(hash_.get("rate", 0))
        self.total = Amount(currency, hash_.get("amount", 0))
        # rate * total = total_in_base
        self.total_in_base = Amount(base_currency, hash_.get("total", 0))

    def as_json(self):
        return {
                "fee_rate": self.fee_rate,
                "date": self.date,
                "action": self.action,
                "total": self.total.value,
                "currency": self.currency,
                "total_in_base": self.total_in_base.value,
                "base_currency": self.base_currency
                }

    def __repr__(self):
        if self.fee_rate > 0:
            fee_rate = " fee: {}%".format(self.fee_rate * 100)
        else:
            fee_rate = ""
        if self.date is None:
            date = "No date"
        else:
            date = self.date
        return "Mouvement({} ; {} {} ({}){})".format(
                date, self.action, self.total, self.total_in_base,
                fee_rate)