path: root/tests/test_market.py

from .helper import *
import market, store, portfolio, dbs
import datetime

@unittest.skipUnless("unit" in limits, "Unit skipped")
class MarketTest(WebMockTestCase):
    def setUp(self):
        super().setUp()

        self.ccxt = mock.Mock(spec=market.ccxt.poloniexE)

    def test_values(self):
        m = market.Market(self.ccxt, self.market_args())

        self.assertEqual(self.ccxt, m.ccxt)
        self.assertFalse(m.debug)
        self.assertIsInstance(m.report, market.ReportStore)
        self.assertIsInstance(m.trades, market.TradeStore)
        self.assertIsInstance(m.balances, market.BalanceStore)
        self.assertEqual(m, m.report.market)
        self.assertEqual(m, m.trades.market)
        self.assertEqual(m, m.balances.market)
        self.assertEqual(m, m.ccxt._market)

        m = market.Market(self.ccxt, self.market_args(debug=True))
        self.assertTrue(m.debug)

        m = market.Market(self.ccxt, self.market_args(debug=False))
        self.assertFalse(m.debug)

        with mock.patch("market.ReportStore") as report_store:
            with self.subTest(quiet=False):
                m = market.Market(self.ccxt, self.market_args(quiet=False))
                report_store.assert_called_with(m, verbose_print=True)
                report_store().log_market.assert_called_once()
            report_store.reset_mock()
            with self.subTest(quiet=True):
                m = market.Market(self.ccxt, self.market_args(quiet=True))
                report_store.assert_called_with(m, verbose_print=False)
                report_store().log_market.assert_called_once()

    @mock.patch("market.ccxt")
    def test_from_config(self, ccxt):
        with mock.patch("market.ReportStore"):
            ccxt.poloniexE.return_value = self.ccxt

            m = market.Market.from_config({"key": "key", "secred": "secret"}, self.market_args())

            self.assertEqual(self.ccxt, m.ccxt)

        m = market.Market.from_config({"key": "key", "secred": "secret"}, self.market_args(debug=True))
        self.assertEqual(True, m.debug)

    def test_get_tickers(self):
        self.ccxt.fetch_tickers.side_effect = [
                "tickers",
                market.NotSupported
                ]

        m = market.Market(self.ccxt, self.market_args())
        self.assertEqual("tickers", m.get_tickers())
        self.assertEqual("tickers", m.get_tickers())
        self.ccxt.fetch_tickers.assert_called_once()

        self.assertIsNone(m.get_tickers(refresh=self.time.time()))

    def test_get_ticker(self):
        with self.subTest(get_tickers=True):
            self.ccxt.fetch_tickers.return_value = {
                    "ETH/ETC": { "bid": 1, "ask": 3 },
                    "XVG/ETH": { "bid": 10, "ask": 40 },
                    }
            m = market.Market(self.ccxt, self.market_args())

            ticker = m.get_ticker("ETH", "ETC")
            self.assertEqual(1, ticker["bid"])
            self.assertEqual(3, ticker["ask"])
            self.assertEqual(2, ticker["average"])
            self.assertFalse(ticker["inverted"])

            ticker = m.get_ticker("ETH", "XVG")
            self.assertEqual(0.0625, ticker["average"])
            self.assertTrue(ticker["inverted"])
            self.assertIn("original", ticker)
            self.assertEqual(10, ticker["original"]["bid"])
            self.assertEqual(25, ticker["original"]["average"])

            ticker = m.get_ticker("XVG", "XMR")
            self.assertIsNone(ticker)

        with self.subTest(get_tickers=False):
            self.ccxt.fetch_tickers.return_value = None
            self.ccxt.fetch_ticker.side_effect = [
                    { "bid": 1, "ask": 3 },
                    market.ExchangeError("foo"),
                    { "bid": 10, "ask": 40 },
                    market.ExchangeError("foo"),
                    market.ExchangeError("foo"),
                    ]

            m = market.Market(self.ccxt, self.market_args())

            ticker = m.get_ticker("ETH", "ETC")
            self.ccxt.fetch_ticker.assert_called_with("ETH/ETC")
            self.assertEqual(1, ticker["bid"])
            self.assertEqual(3, ticker["ask"])
            self.assertEqual(2, ticker["average"])
            self.assertFalse(ticker["inverted"])

            ticker = m.get_ticker("ETH", "XVG")
            self.assertEqual(0.0625, ticker["average"])
            self.assertTrue(ticker["inverted"])
            self.assertIn("original", ticker)
            self.assertEqual(10, ticker["original"]["bid"])
            self.assertEqual(25, ticker["original"]["average"])

            ticker = m.get_ticker("XVG", "XMR")
            self.assertIsNone(ticker)

    def test_fetch_fees(self):
        m = market.Market(self.ccxt, self.market_args())
        self.ccxt.fetch_fees.return_value = "Foo"
        self.assertEqual("Foo", m.fetch_fees())
        self.ccxt.fetch_fees.assert_called_once()
        self.ccxt.reset_mock()
        self.assertEqual("Foo", m.fetch_fees())
        self.ccxt.fetch_fees.assert_not_called()

    @mock.patch.object(market.Portfolio, "repartition")
    @mock.patch.object(market.Market, "get_ticker")
    @mock.patch.object(market.TradeStore, "compute_trades")
    def test_prepare_trades(self, compute_trades, get_ticker, repartition):
        with self.subTest(available_balance_only=False),\
                mock.patch("market.ReportStore"):
            def _get_ticker(c1, c2):
                if c1 == "USDT" and c2 == "BTC":
                    return { "average": D("0.0001") }
                if c1 == "XVG" and c2 == "BTC":
                    return { "average": D("0.000001") }
                self.fail("Should not be called with {}, {}".format(c1, c2))
            get_ticker.side_effect = _get_ticker

            repartition.return_value = {
                    "XEM": (D("0.75"), "long"),
                    "BTC": (D("0.25"), "long"),
                    }
            m = market.Market(self.ccxt, self.market_args())
            self.ccxt.fetch_all_balances.return_value = {
                    "USDT": {
                        "exchange_free": D("10000.0"),
                        "exchange_used": D("0.0"),
                        "exchange_total": D("10000.0"),
                        "total": D("10000.0")
                        },
                    "XVG": {
                        "exchange_free": D("10000.0"),
                        "exchange_used": D("0.0"),
                        "exchange_total": D("10000.0"),
                        "total": D("10000.0")
                        },
                    }

            m.balances.fetch_balances(tag="tag")

            m.prepare_trades()
            compute_trades.assert_called()

            call = compute_trades.call_args
            self.assertEqual(1, call[0][0]["USDT"].value)
            self.assertEqual(D("0.01"), call[0][0]["XVG"].value)
            self.assertEqual(D("0.2525"), call[0][1]["BTC"].value)
            self.assertEqual(D("0.7575"), call[0][1]["XEM"].value)
            m.report.log_stage.assert_called_once_with("prepare_trades",
                    base_currency='BTC', compute_value='average',
                    available_balance_only=False, liquidity='medium',
                    only=None, repartition=None)
            m.report.log_balances.assert_called_once_with(tag="tag", checkpoint=None)

        compute_trades.reset_mock()
        with self.subTest(available_balance_only=True),\
                mock.patch("market.ReportStore"):
            def _get_ticker(c1, c2):
                if c1 == "ZRC" and c2 == "BTC":
                    return { "average": D("0.0001") }
                if c1 == "DOGE" and c2 == "BTC":
                    return { "average": D("0.000001") }
                if c1 == "ETH" and c2 == "BTC":
                    return { "average": D("0.1") }
                if c1 == "FOO" and c2 == "BTC":
                    return { "average": D("0.1") }
                self.fail("Should not be called with {}, {}".format(c1, c2))
            get_ticker.side_effect = _get_ticker

            repartition.return_value = {
                    "DOGE": (D("0.20"), "short"),
                    "BTC": (D("0.20"), "long"),
                    "ETH": (D("0.20"), "long"),
                    "XMR": (D("0.20"), "long"),
                    "FOO": (D("0.20"), "long"),
                    }
            m = market.Market(self.ccxt, self.market_args())
            self.ccxt.fetch_all_balances.return_value = {
                    "ZRC": {
                        "exchange_free": D("2.0"),
                        "exchange_used": D("0.0"),
                        "exchange_total": D("2.0"),
                        "total": D("2.0")
                        },
                    "DOGE": {
                        "exchange_free": D("5.0"),
                        "exchange_used": D("0.0"),
                        "exchange_total": D("5.0"),
                        "total": D("5.0")
                        },
                    "BTC": {
                        "exchange_free": D("0.065"),
                        "exchange_used": D("0.02"),
                        "exchange_total": D("0.085"),
                        "margin_available": D("0.035"),
                        "margin_in_position": D("0.01"),
                        "margin_total": D("0.045"),
                        "total": D("0.13")
                        },
                    "ETH": {
                        "exchange_free": D("1.0"),
                        "exchange_used": D("0.0"),
                        "exchange_total": D("1.0"),
                        "total": D("1.0")
                        },
                    "FOO": {
                        "exchange_free": D("0.1"),
                        "exchange_used": D("0.0"),
                        "exchange_total": D("0.1"),
                        "total": D("0.1"),
                        },
                    }

            m.balances.fetch_balances(tag="tag")
            m.prepare_trades(available_balance_only=True)
            compute_trades.assert_called_once()

            call = compute_trades.call_args[0]
            values_in_base = call[0]
            new_repartition = call[1]

            self.assertEqual(portfolio.Amount("BTC", "-0.025"),
                    new_repartition["DOGE"] - values_in_base["DOGE"])
            self.assertEqual(0,
                    new_repartition["ETH"] - values_in_base["ETH"])
            self.assertIsNone(new_repartition.get("ZRC"))
            self.assertEqual(portfolio.Amount("BTC", "0.025"),
                    new_repartition["XMR"])
            self.assertEqual(portfolio.Amount("BTC", "0.015"),
                    new_repartition["FOO"] - values_in_base["FOO"])

        compute_trades.reset_mock()
        with self.subTest(available_balance_only=True, balance=0),\
                mock.patch("market.ReportStore"):
            def _get_ticker(c1, c2):
                if c1 == "ETH" and c2 == "BTC":
                    return { "average": D("0.1") }
                self.fail("Should not be called with {}, {}".format(c1, c2))
            get_ticker.side_effect = _get_ticker

            repartition.return_value = {
                    "BTC": (D("0.5"), "long"),
                    "ETH": (D("0.5"), "long"),
                    }
            m = market.Market(self.ccxt, self.market_args())
            self.ccxt.fetch_all_balances.return_value = {
                    "ETH": {
                        "exchange_free": D("1.0"),
                        "exchange_used": D("0.0"),
                        "exchange_total": D("1.0"),
                        "total": D("1.0")
                        },
                    }

            m.balances.fetch_balances(tag="tag")
            m.prepare_trades(available_balance_only=True)
            compute_trades.assert_called_once()

            call = compute_trades.call_args[0]
            values_in_base = call[0]
            new_repartition = call[1]

            self.assertEqual(new_repartition["ETH"], values_in_base["ETH"])

    @mock.patch.object(market.time, "sleep")
    @mock.patch.object(market.TradeStore, "all_orders")
    def test_follow_orders(self, all_orders, time_mock):
        for debug, sleep in [
                (False, None), (True, None),
                (False, 12), (True, 12)]:
            with self.subTest(sleep=sleep, debug=debug), \
                    mock.patch("market.ReportStore"):
                m = market.Market(self.ccxt, self.market_args(debug=debug))

                order_mock1 = mock.Mock()
                order_mock2 = mock.Mock()
                order_mock3 = mock.Mock()
                all_orders.side_effect = [
                        [order_mock1, order_mock2],
                        [order_mock1, order_mock2],

                        [order_mock1, order_mock3],
                        [order_mock1, order_mock3],

                        [order_mock1, order_mock3],
                        [order_mock1, order_mock3],

                        []
                        ]

                order_mock1.get_status.side_effect = ["open", "open", "closed"]
                order_mock2.get_status.side_effect = ["open"]
                order_mock3.get_status.side_effect = ["open", "closed"]

                order_mock1.trade = mock.Mock()
                order_mock2.trade = mock.Mock()
                order_mock3.trade = mock.Mock()

                m.follow_orders(sleep=sleep)

                order_mock1.trade.update_order.assert_any_call(order_mock1, 1)
                order_mock1.trade.update_order.assert_any_call(order_mock1, 2)
                self.assertEqual(2, order_mock1.trade.update_order.call_count)
                self.assertEqual(3, order_mock1.get_status.call_count)

                order_mock2.trade.update_order.assert_any_call(order_mock2, 1)
                self.assertEqual(1, order_mock2.trade.update_order.call_count)
                self.assertEqual(1, order_mock2.get_status.call_count)

                order_mock3.trade.update_order.assert_any_call(order_mock3, 2)
                self.assertEqual(1, order_mock3.trade.update_order.call_count)
                self.assertEqual(2, order_mock3.get_status.call_count)
                m.report.log_stage.assert_called()
                calls = [
                        mock.call("follow_orders_begin"),
                        mock.call("follow_orders_tick_1"),
                        mock.call("follow_orders_tick_2"),
                        mock.call("follow_orders_tick_3"),
                        mock.call("follow_orders_end"),
                        ]
                m.report.log_stage.assert_has_calls(calls)
                m.report.log_orders.assert_called()
                self.assertEqual(3, m.report.log_orders.call_count)
                calls = [
                        mock.call([order_mock1, order_mock2], tick=1),
                        mock.call([order_mock1, order_mock3], tick=2),
                        mock.call([order_mock1, order_mock3], tick=3),
                        ]
                m.report.log_orders.assert_has_calls(calls)
                calls = [
                        mock.call(order_mock1, 3, finished=True),
                        mock.call(order_mock3, 3, finished=True),
                        ]
                m.report.log_order.assert_has_calls(calls)

                if sleep is None:
                    if debug:
                        m.report.log_debug_action.assert_called_with("Set follow_orders tick to 7s")
                        time_mock.assert_called_with(7)
                    else:
                        time_mock.assert_called_with(30)
                else:
                    time_mock.assert_called_with(sleep)

        with self.subTest("disappearing order"), \
                mock.patch("market.ReportStore"):
            all_orders.reset_mock()
            m = market.Market(self.ccxt, self.market_args())

            order_mock1 = mock.Mock()
            order_mock2 = mock.Mock()
            all_orders.side_effect = [
                    [order_mock1, order_mock2],
                    [order_mock1, order_mock2],

                    [order_mock1, order_mock2],
                    [order_mock1, order_mock2],

                    []
                    ]

            order_mock1.get_status.side_effect = ["open", "closed"]
            order_mock2.get_status.side_effect = ["open", "error_disappeared"]

            order_mock1.trade = mock.Mock()
            trade_mock = mock.Mock()
            order_mock2.trade = trade_mock

            trade_mock.tick_actions_recreate.return_value = "tick1"
            new_order_mock = mock.Mock()
            trade_mock.prepare_order.return_value = new_order_mock

            m.follow_orders()

            trade_mock.tick_actions_recreate.assert_called_once_with(2)
            trade_mock.prepare_order.assert_called_once_with(compute_value="tick1")
            m.report.log_error.assert_called_once_with("follow_orders", message=mock.ANY)
            m.report.log_order.assert_called_with(order_mock2, 2, new_order=new_order_mock)
            new_order_mock.run.assert_called_once_with()

        with self.subTest("disappearing order no action to do"), \
                mock.patch("market.ReportStore"):
            all_orders.reset_mock()
            m = market.Market(self.ccxt, self.market_args())

            order_mock1 = mock.Mock()
            order_mock2 = mock.Mock()
            all_orders.side_effect = [
                    [order_mock1, order_mock2],
                    [order_mock1, order_mock2],

                    [order_mock1, order_mock2],
                    [order_mock1, order_mock2],

                    []
                    ]

            order_mock1.get_status.side_effect = ["open", "closed"]
            order_mock2.get_status.side_effect = ["open", "error_disappeared"]

            order_mock1.trade = mock.Mock()
            trade_mock = mock.Mock()
            order_mock2.trade = trade_mock

            trade_mock.tick_actions_recreate.return_value = "tick1"
            trade_mock.prepare_order.return_value = None

            m.follow_orders()

            trade_mock.tick_actions_recreate.assert_called_once_with(2)
            trade_mock.prepare_order.assert_called_once_with(compute_value="tick1")
            m.report.log_error.assert_called_once_with("follow_orders", message=mock.ANY)
            m.report.log_order.assert_called_with(order_mock2, 2, finished=True)

    @mock.patch.object(market.BalanceStore, "fetch_balances")
    def test_move_balance(self, fetch_balances):
        for debug in [True, False]:
            with self.subTest(debug=debug),\
                    mock.patch("market.ReportStore"):
                m = market.Market(self.ccxt, self.market_args(debug=debug))

                value_from = portfolio.Amount("BTC", "1.0")
                value_from.linked_to = portfolio.Amount("ETH", "10.0")
                value_to = portfolio.Amount("BTC", "10.0")
                trade1 = portfolio.Trade(value_from, value_to, "ETH", m)

                value_from = portfolio.Amount("BTC", "0.0")
                value_from.linked_to = portfolio.Amount("ETH", "0.0")
                value_to = portfolio.Amount("BTC", "-3.0")
                trade2 = portfolio.Trade(value_from, value_to, "ETH", m)

                value_from = portfolio.Amount("USDT", "0.0")
                value_from.linked_to = portfolio.Amount("XVG", "0.0")
                value_to = portfolio.Amount("USDT", "-50.0")
                trade3 = portfolio.Trade(value_from, value_to, "XVG", m)

                m.trades.all = [trade1, trade2, trade3]
                balance1 = portfolio.Balance("BTC", { "margin_in_position": "0", "margin_available": "0" })
                balance2 = portfolio.Balance("USDT", { "margin_in_position": "100", "margin_available": "50" })
                balance3 = portfolio.Balance("ETC", { "margin_in_position": "10", "margin_available": "15" })
                m.balances.all = {"BTC": balance1, "USDT": balance2, "ETC": balance3}

                m.move_balances()

                fetch_balances.assert_called_with()
                m.report.log_move_balances.assert_called_once()

                if debug:
                    m.report.log_debug_action.assert_called()
                    self.assertEqual(3, m.report.log_debug_action.call_count)
                else:
                    self.ccxt.transfer_balance.assert_any_call("BTC", 3, "exchange", "margin")
                    self.ccxt.transfer_balance.assert_any_call("USDT", 100, "exchange", "margin")
                    self.ccxt.transfer_balance.assert_any_call("ETC", 5, "margin", "exchange")

        m.report.reset_mock()
        fetch_balances.reset_mock()
        with self.subTest(retry=True):
            with mock.patch("market.ReportStore"):
                m = market.Market(self.ccxt, self.market_args())

                value_from = portfolio.Amount("BTC", "0.0")
                value_from.linked_to = portfolio.Amount("ETH", "0.0")
                value_to = portfolio.Amount("BTC", "-3.0")
                trade = portfolio.Trade(value_from, value_to, "ETH", m)

                m.trades.all = [trade]
                balance = portfolio.Balance("BTC", { "margin_in_position": "0", "margin_available": "0" })
                m.balances.all = {"BTC": balance}

                m.ccxt.transfer_balance.side_effect = [
                        market.ccxt.RequestTimeout,
                        market.ccxt.InvalidNonce,
                        True
                        ]
                m.move_balances()
                self.ccxt.transfer_balance.assert_has_calls([
                    mock.call("BTC", 3, "exchange", "margin"),
                    mock.call("BTC", 3, "exchange", "margin"),
                    mock.call("BTC", 3, "exchange", "margin")
                    ])
                self.assertEqual(3, fetch_balances.call_count)
                m.report.log_error.assert_called_with(mock.ANY, message="Retrying", exception=mock.ANY)
                self.assertEqual(3, m.report.log_move_balances.call_count)

        self.ccxt.transfer_balance.reset_mock()
        m.report.reset_mock()
        fetch_balances.reset_mock()
        with self.subTest(retry=True, too_much=True):
            with mock.patch("market.ReportStore"):
                m = market.Market(self.ccxt, self.market_args())

                value_from = portfolio.Amount("BTC", "0.0")
                value_from.linked_to = portfolio.Amount("ETH", "0.0")
                value_to = portfolio.Amount("BTC", "-3.0")
                trade = portfolio.Trade(value_from, value_to, "ETH", m)

                m.trades.all = [trade]
                balance = portfolio.Balance("BTC", { "margin_in_position": "0", "margin_available": "0" })
                m.balances.all = {"BTC": balance}

                m.ccxt.transfer_balance.side_effect = [
                        market.ccxt.RequestTimeout,
                        market.ccxt.RequestTimeout,
                        market.ccxt.RequestTimeout,
                        market.ccxt.RequestTimeout,
                        market.ccxt.RequestTimeout,
                        ]
                with self.assertRaises(market.ccxt.RequestTimeout):
                    m.move_balances()

        self.ccxt.transfer_balance.reset_mock()
        m.report.reset_mock()
        fetch_balances.reset_mock()
        with self.subTest(retry=True, partial_result=True):
            with mock.patch("market.ReportStore"):
                m = market.Market(self.ccxt, self.market_args())

                value_from = portfolio.Amount("BTC", "1.0")
                value_from.linked_to = portfolio.Amount("ETH", "10.0")
                value_to = portfolio.Amount("BTC", "10.0")
                trade1 = portfolio.Trade(value_from, value_to, "ETH", m)

                value_from = portfolio.Amount("BTC", "0.0")
                value_from.linked_to = portfolio.Amount("ETH", "0.0")
                value_to = portfolio.Amount("BTC", "-3.0")
                trade2 = portfolio.Trade(value_from, value_to, "ETH", m)

                value_from = portfolio.Amount("USDT", "0.0")
                value_from.linked_to = portfolio.Amount("XVG", "0.0")
                value_to = portfolio.Amount("USDT", "-50.0")
                trade3 = portfolio.Trade(value_from, value_to, "XVG", m)

                m.trades.all = [trade1, trade2, trade3]
                balance1 = portfolio.Balance("BTC", { "margin_in_position": "0", "margin_available": "0" })
                balance2 = portfolio.Balance("USDT", { "margin_in_position": "100", "margin_available": "50" })
                balance3 = portfolio.Balance("ETC", { "margin_in_position": "10", "margin_available": "15" })
                m.balances.all = {"BTC": balance1, "USDT": balance2, "ETC": balance3}

                call_counts = { "BTC": 0, "USDT": 0, "ETC": 0 }
                def _transfer_balance(currency, amount, from_, to_):
                    call_counts[currency] += 1
                    if currency == "BTC":
                        m.balances.all["BTC"] = portfolio.Balance("BTC", { "margin_in_position": "0", "margin_available": "3" })
                    if currency == "USDT":
                        if call_counts["USDT"] == 1:
                            raise market.ccxt.RequestTimeout
                        else:
                            m.balances.all["USDT"] = portfolio.Balance("USDT", { "margin_in_position": "100", "margin_available": "150" })
                    if currency == "ETC":
                            m.balances.all["ETC"] = portfolio.Balance("ETC", { "margin_in_position": "10", "margin_available": "10" })


                m.ccxt.transfer_balance.side_effect = _transfer_balance

                m.move_balances()
                self.ccxt.transfer_balance.assert_has_calls([
                    mock.call("BTC", 3, "exchange", "margin"),
                    mock.call('USDT', 100, 'exchange', 'margin'),
                    mock.call('USDT', 100, 'exchange', 'margin'),
                    mock.call("ETC", 5, "margin", "exchange")
                    ])
                self.assertEqual(2, fetch_balances.call_count)
                m.report.log_error.assert_called_with(mock.ANY, message="Retrying", exception=mock.ANY)
                self.assertEqual(2, m.report.log_move_balances.call_count)
                m.report.log_move_balances.asser_has_calls([
                    mock.call(
                        {
                            'BTC': portfolio.Amount("BTC", "3"),
                            'USDT': portfolio.Amount("USDT", "150"),
                            'ETC': portfolio.Amount("ETC", "10"),
                            },
                        {
                            'BTC': portfolio.Amount("BTC", "3"),
                            'USDT': portfolio.Amount("USDT", "100"),
                            }),
                    mock.call(
                        {
                            'BTC': portfolio.Amount("BTC", "3"),
                            'USDT': portfolio.Amount("USDT", "150"),
                            'ETC': portfolio.Amount("ETC", "10"),
                            },
                        {
                            'BTC': portfolio.Amount("BTC", "0"),
                            'USDT': portfolio.Amount("USDT", "100"),
                            'ETC': portfolio.Amount("ETC", "-5"),
                            }),
                    ])


    def test_store_file_report(self):
        file_open = mock.mock_open()
        m = market.Market(self.ccxt,
                self.market_args(report_path="present"), user_id=1)
        with self.subTest(file="present"),\
                mock.patch("market.open", file_open),\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(market, "datetime") as time_mock:

            report.print_logs = [[time_mock.now(), "Foo"], [time_mock.now(), "Bar"]]
            report.to_json.return_value = "json_content"

            m.store_file_report(datetime.datetime(2018, 2, 25))

            file_open.assert_any_call("present/2018-02-25T00:00:00_1.json", "w")
            file_open.assert_any_call("present/2018-02-25T00:00:00_1.log", "w")
            file_open().write.assert_any_call("json_content")
            file_open().write.assert_any_call("Foo\nBar")
            m.report.to_json.assert_called_once_with()

        m = market.Market(self.ccxt, self.market_args(report_path="error"), user_id=1)
        with self.subTest(file="error"),\
                mock.patch("market.open") as file_open,\
                mock.patch.object(m, "report") as report,\
                mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock:
            file_open.side_effect = FileNotFoundError

            m.store_file_report(datetime.datetime(2018, 2, 25))

            self.assertRegex(stdout_mock.getvalue(), "impossible to store report file: FileNotFoundError;")

    @mock.patch.object(dbs, "psql")
    def test_store_database_report(self, psql):
        cursor_mock = mock.MagicMock()

        psql.cursor.return_value = cursor_mock
        m = market.Market(self.ccxt, self.market_args(),
                pg_config={"config": "pg_config"}, user_id=1)
        cursor_mock.fetchone.return_value = [42]

        with self.subTest(error=False),\
                mock.patch.object(m, "report") as report:
            report.to_json_array.return_value = [
                    ("date1", "type1", "payload1"),
                    ("date2", "type2", "payload2"),
                    ]
            m.store_database_report(datetime.datetime(2018, 3, 24))
            psql.assert_has_calls([
                mock.call.cursor(),
                mock.call.cursor().execute('INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id;', (datetime.datetime(2018, 3, 24), None, False)),
                mock.call.cursor().fetchone(),
                mock.call.cursor().execute('INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);', ('date1', 42, 'type1', 'payload1')),
                mock.call.cursor().execute('INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);', ('date2', 42, 'type2', 'payload2')),
                mock.call.commit(),
                mock.call.cursor().close(),
                ])

        with self.subTest(error=True),\
                mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock:
            psql.cursor.side_effect = Exception("Bouh")
            m.store_database_report(datetime.datetime(2018, 3, 24))
            self.assertEqual(stdout_mock.getvalue(), "impossible to store report to database: Exception; Bouh\n")

    @mock.patch.object(dbs, "redis")
    def test_store_redis_report(self, redis):
        m = market.Market(self.ccxt, self.market_args(),
                redis_config={"config": "redis_config"}, market_id=1)

        with self.subTest(error=False),\
                mock.patch.object(m, "report") as report:
            report.to_json_redis.return_value = [
                    ("type1", "payload1"),
                    ("type2", "payload2"),
                    ]
            m.store_redis_report(datetime.datetime(2018, 3, 24))
            redis.assert_has_calls([
                mock.call.set("/cryptoportfolio/1/2018-03-24T00:00:00/type1", "payload1", ex=31*24*60*60),
                mock.call.set("/cryptoportfolio/1/latest/type1", "payload1"),
                mock.call.set("/cryptoportfolio/1/2018-03-24T00:00:00/type2", "payload2", ex=31*24*60*60),
                mock.call.set("/cryptoportfolio/1/latest/type2", "payload2"),
                mock.call.set("/cryptoportfolio/1/latest/date", "2018-03-24T00:00:00"),
                ])

        redis.reset_mock()
        with self.subTest(error=True),\
                mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock:
            redis.set.side_effect = Exception("Bouh")
            m.store_redis_report(datetime.datetime(2018, 3, 24))
            self.assertEqual(stdout_mock.getvalue(), "impossible to store report to redis: Exception; Bouh\n")

    def test_store_report(self):
        m = market.Market(self.ccxt, self.market_args(report_db=False), user_id=1)
        with self.subTest(file=None, pg_connected=None),\
                mock.patch.object(dbs, "psql_connected") as psql,\
                mock.patch.object(dbs, "redis_connected") as redis,\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(m, "store_database_report") as db_report,\
                mock.patch.object(m, "store_redis_report") as redis_report,\
                mock.patch.object(m, "store_file_report") as file_report:
            psql.return_value = False
            redis.return_value = False
            m.store_report()
            report.merge.assert_called_with(store.Portfolio.report)

            file_report.assert_not_called()
            db_report.assert_not_called()
            redis_report.assert_not_called()

        report.reset_mock()
        m = market.Market(self.ccxt, self.market_args(report_db=False, report_path="present"), user_id=1)
        with self.subTest(file="present", pg_connected=None),\
                mock.patch.object(dbs, "psql_connected") as psql,\
                mock.patch.object(dbs, "redis_connected") as redis,\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(m, "store_file_report") as file_report,\
                mock.patch.object(m, "store_redis_report") as redis_report,\
                mock.patch.object(m, "store_database_report") as db_report,\
                mock.patch.object(market.datetime, "datetime") as time_mock:
            psql.return_value = False
            redis.return_value = False
            time_mock.now.return_value = datetime.datetime(2018, 2, 25)

            m.store_report()

            report.merge.assert_called_with(store.Portfolio.report)
            file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
            db_report.assert_not_called()
            redis_report.assert_not_called()

        report.reset_mock()
        m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), user_id=1)
        with self.subTest(file="present", pg_connected=None, report_db=True),\
                mock.patch.object(dbs, "psql_connected") as psql,\
                mock.patch.object(dbs, "redis_connected") as redis,\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(m, "store_file_report") as file_report,\
                mock.patch.object(m, "store_redis_report") as redis_report,\
                mock.patch.object(m, "store_database_report") as db_report,\
                mock.patch.object(market.datetime, "datetime") as time_mock:
            psql.return_value = False
            redis.return_value = False
            time_mock.now.return_value = datetime.datetime(2018, 2, 25)

            m.store_report()

            report.merge.assert_called_with(store.Portfolio.report)
            file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
            db_report.assert_not_called()
            redis_report.assert_not_called()

        report.reset_mock()
        m = market.Market(self.ccxt, self.market_args(report_db=True), user_id=1)
        with self.subTest(file=None, pg_connected=True),\
                mock.patch.object(dbs, "psql_connected") as psql,\
                mock.patch.object(dbs, "redis_connected") as redis,\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(m, "store_file_report") as file_report,\
                mock.patch.object(m, "store_redis_report") as redis_report,\
                mock.patch.object(m, "store_database_report") as db_report,\
                mock.patch.object(market.datetime, "datetime") as time_mock:
            psql.return_value = True
            redis.return_value = False
            time_mock.now.return_value = datetime.datetime(2018, 2, 25)

            m.store_report()

            report.merge.assert_called_with(store.Portfolio.report)
            file_report.assert_not_called()
            db_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
            redis_report.assert_not_called()

        report.reset_mock()
        m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"),
                user_id=1)
        with self.subTest(file="present", pg_connected=True),\
                mock.patch.object(dbs, "psql_connected") as psql,\
                mock.patch.object(dbs, "redis_connected") as redis,\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(m, "store_file_report") as file_report,\
                mock.patch.object(m, "store_redis_report") as redis_report,\
                mock.patch.object(m, "store_database_report") as db_report,\
                mock.patch.object(market.datetime, "datetime") as time_mock:
            psql.return_value = True
            redis.return_value = False
            time_mock.now.return_value = datetime.datetime(2018, 2, 25)

            m.store_report()

            report.merge.assert_called_with(store.Portfolio.report)
            file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
            db_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
            redis_report.assert_not_called()

        report.reset_mock()
        m = market.Market(self.ccxt, self.market_args(report_redis=False),
                user_id=1)
        with self.subTest(redis_connected=True, report_redis=False),\
                mock.patch.object(dbs, "psql_connected") as psql,\
                mock.patch.object(dbs, "redis_connected") as redis,\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(m, "store_file_report") as file_report,\
                mock.patch.object(m, "store_redis_report") as redis_report,\
                mock.patch.object(m, "store_database_report") as db_report,\
                mock.patch.object(market.datetime, "datetime") as time_mock:
            psql.return_value = False
            redis.return_value = True
            time_mock.now.return_value = datetime.datetime(2018, 2, 25)

            m.store_report()
            redis_report.assert_not_called()

        report.reset_mock()
        m = market.Market(self.ccxt, self.market_args(report_redis=True),
                user_id=1)
        with self.subTest(redis_connected=False, report_redis=True),\
                mock.patch.object(dbs, "psql_connected") as psql,\
                mock.patch.object(dbs, "redis_connected") as redis,\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(m, "store_file_report") as file_report,\
                mock.patch.object(m, "store_redis_report") as redis_report,\
                mock.patch.object(m, "store_database_report") as db_report,\
                mock.patch.object(market.datetime, "datetime") as time_mock:
            psql.return_value = False
            redis.return_value = False
            time_mock.now.return_value = datetime.datetime(2018, 2, 25)

            m.store_report()
            redis_report.assert_not_called()

        report.reset_mock()
        m = market.Market(self.ccxt, self.market_args(report_redis=True),
                user_id=1)
        with self.subTest(redis_connected=True, report_redis=True),\
                mock.patch.object(dbs, "psql_connected") as psql,\
                mock.patch.object(dbs, "redis_connected") as redis,\
                mock.patch.object(m, "report") as report,\
                mock.patch.object(m, "store_file_report") as file_report,\
                mock.patch.object(m, "store_redis_report") as redis_report,\
                mock.patch.object(m, "store_database_report") as db_report,\
                mock.patch.object(market.datetime, "datetime") as time_mock:
            psql.return_value = False
            redis.return_value = True
            time_mock.now.return_value = datetime.datetime(2018, 2, 25)

            m.store_report()
            redis_report.assert_called_once_with(datetime.datetime(2018, 2, 25))

    def test_print_tickers(self):
        m = market.Market(self.ccxt, self.market_args())

        with mock.patch.object(m.balances, "in_currency") as in_currency,\
                mock.patch.object(m.report, "log_stage") as log_stage,\
                mock.patch.object(m.balances, "fetch_balances") as fetch_balances,\
                mock.patch.object(m.report, "print_log") as print_log:

            in_currency.return_value = {
                    "BTC": portfolio.Amount("BTC", "0.65"),
                    "ETH": portfolio.Amount("BTC", "0.3"),
                    }

            m.print_tickers()

            print_log.assert_has_calls([
                mock.call("total:"),
                mock.call(portfolio.Amount("BTC", "0.95")),
                ])

    @mock.patch("market.Processor.process")
    @mock.patch("market.ReportStore.log_error")
    @mock.patch("market.Market.store_report")
    def test_process(self, store_report, log_error, process):
        m = market.Market(self.ccxt, self.market_args(), options={"foo": "bar"})
        with self.subTest(actions=[], before=False, after=False):
            m.process([])

            process.assert_not_called()
            store_report.assert_called_once()
            log_error.assert_not_called()

        process.reset_mock()
        log_error.reset_mock()
        store_report.reset_mock()
        with self.subTest(before=True, after=False):
            m.process(["foo"], before=True)

            process.assert_called_once_with("foo", options={"foo": "bar"}, steps="before")
            store_report.assert_called_once()
            log_error.assert_not_called()

        process.reset_mock()
        log_error.reset_mock()
        store_report.reset_mock()
        with self.subTest(before=False, after=True):
            m.process(["sell_all"], after=True)

            process.assert_called_once_with("sell_all", options={"foo": "bar"}, steps="after")
            store_report.assert_called_once()
            log_error.assert_not_called()

        process.reset_mock()
        log_error.reset_mock()
        store_report.reset_mock()
        with self.subTest(before=False, after=False):
            m.process(["foo"])

            process.assert_called_once_with("foo", options={"foo": "bar"}, steps="all")
            store_report.assert_called_once()
            log_error.assert_not_called()

        process.reset_mock()
        log_error.reset_mock()
        store_report.reset_mock()
        with self.subTest(before=True, after=True):
            m.process(["sell_all"], before=True, after=True)

            process.assert_called_once_with("sell_all", options={"foo": "bar"}, steps="all")
            store_report.assert_called_once()
            log_error.assert_not_called()

        process.reset_mock()
        log_error.reset_mock()
        store_report.reset_mock()
        with self.subTest(authentication_error=True):
            m.ccxt.check_required_credentials.side_effect = market.ccxt.AuthenticationError

            m.process(["some_action"], before=True)
            log_error.assert_called_with("market_authentication", message="Impossible to authenticate to market")
            store_report.assert_called_once()

        m.ccxt.check_required_credentials.side_effect = True
        process.reset_mock()
        log_error.reset_mock()
        store_report.reset_mock()
        with self.subTest(unhandled_exception=True):
            process.side_effect = Exception("bouh")

            m.process(["some_action"], before=True)
            log_error.assert_called_with("market_process", exception=mock.ANY, message=mock.ANY)
            store_report.assert_called_once()
 

@unittest.skipUnless("unit" in limits, "Unit skipped")
class ProcessorTest(WebMockTestCase):
    def test_values(self):
        processor = market.Processor(self.m)

        self.assertEqual(self.m, processor.market)

    def test_run_action(self):
        processor = market.Processor(self.m)

        with mock.patch.object(processor, "parse_args") as parse_args:
            method_mock = mock.Mock()
            parse_args.return_value = [method_mock, { "foo": "bar" }]

            processor.run_action("foo", "bar", "baz")

            parse_args.assert_called_with("foo", "bar", "baz")

            method_mock.assert_called_with(foo="bar")

            processor.run_action("wait_for_recent", "bar", "baz")

            method_mock.assert_called_with(foo="bar")

    def test_select_step(self):
        processor = market.Processor(self.m)

        scenario = processor.scenarios["sell_all"]

        self.assertEqual(scenario, processor.select_steps(scenario, "all"))
        self.assertEqual(["all_sell"], list(map(lambda x: x["name"], processor.select_steps(scenario, "before"))))
        self.assertEqual(["wait", "all_buy"], list(map(lambda x: x["name"], processor.select_steps(scenario, "after"))))
        self.assertEqual(["wait"], list(map(lambda x: x["name"], processor.select_steps(scenario, 2))))
        self.assertEqual(["wait"], list(map(lambda x: x["name"], processor.select_steps(scenario, "wait"))))

        with self.assertRaises(TypeError):
            processor.select_steps(scenario, ["wait"])

    def test_can_process(self):
        processor = market.Processor(self.m)

        with self.subTest(True):
            self.assertTrue(processor.can_process("sell_all"))

        with self.subTest(False):
            self.assertFalse(processor.can_process("unknown_action"))

    @mock.patch("market.Processor.process_step")
    def test_process(self, process_step):
        with self.subTest("unknown action"):
            processor = market.Processor(self.m)
            with self.assertRaises(TypeError):
                processor.process("unknown_action")

        with self.subTest("nominal case"):
            processor = market.Processor(self.m)

            processor.process("sell_all", options="bar")
            self.assertEqual(3, process_step.call_count)

            steps = list(map(lambda x: x[1][1]["name"], process_step.mock_calls))
            scenario_names = list(map(lambda x: x[1][0], process_step.mock_calls))
            kwargs = list(map(lambda x: x[1][2], process_step.mock_calls))
            self.assertEqual(["all_sell", "wait", "all_buy"], steps)
            self.assertEqual(["sell_all", "sell_all", "sell_all"], scenario_names)
            self.assertEqual(["bar", "bar", "bar"], kwargs)

            process_step.reset_mock()

            processor.process("sell_needed", steps=["before", "after"])
            self.assertEqual(4, process_step.call_count)

    def test_method_arguments(self):
        ccxt = mock.Mock(spec=market.ccxt.poloniexE)
        m = market.Market(ccxt, self.market_args())

        processor = market.Processor(m)

        method, arguments = processor.method_arguments("wait_for_recent")
        self.assertEqual(market.Portfolio.wait_for_recent, method)
        self.assertEqual(["delta"], arguments)

        method, arguments = processor.method_arguments("prepare_trades")
        self.assertEqual(m.prepare_trades, method)
        self.assertEqual(['base_currency', 'liquidity', 'compute_value', 'repartition', 'only', 'available_balance_only'], arguments)

        method, arguments = processor.method_arguments("prepare_orders")
        self.assertEqual(m.trades.prepare_orders, method)

        method, arguments = processor.method_arguments("move_balances")
        self.assertEqual(m.move_balances, method)

        method, arguments = processor.method_arguments("run_orders")
        self.assertEqual(m.trades.run_orders, method)

        method, arguments = processor.method_arguments("follow_orders")
        self.assertEqual(m.follow_orders, method)

        method, arguments = processor.method_arguments("close_trades")
        self.assertEqual(m.trades.close_trades, method)

        method, arguments = processor.method_arguments("print_tickers")
        self.assertEqual(m.print_tickers, method)

        method, arguments = processor.method_arguments("fetch_balances")
        self.assertEqual(m.balances.fetch_balances, method)

    def test_process_step(self):
        processor = market.Processor(self.m)

        with mock.patch.object(processor, "run_action") as run_action:
            step = processor.scenarios["sell_needed"][2]

            processor.process_step("foo", step, {"foo":"bar"})

            self.m.report.log_stage.assert_has_calls([
                mock.call("process_foo__2_sell_begin"),
                mock.call("process_foo__2_sell_end"),
                ])

            self.assertEqual(7, run_action.call_count)

            run_action.assert_has_calls([
                mock.call('fetch_balances', {}, {'foo': 'bar', 'tag': 'process_foo__2_sell_begin'}),
                mock.call('prepare_trades', {}, {'foo': 'bar'}),
                mock.call('prepare_orders', {'only': 'dispose', 'compute_value': 'average'}, {'foo': 'bar'}),
                mock.call('run_orders', {}, {'foo': 'bar'}),
                mock.call('follow_orders', {}, {'foo': 'bar'}),
                mock.call('close_trades', {}, {'foo': 'bar'}),
                mock.call('fetch_balances', {}, {'foo': 'bar', 'tag': 'process_foo__2_sell_end'}),
                ])

        self.m.reset_mock()
        with mock.patch.object(processor, "run_action") as run_action:
            step = processor.scenarios["sell_needed"][0]

            processor.process_step("foo", step, {"foo":"bar"})

            self.m.report.log_stage.assert_has_calls([
                mock.call("process_foo__0_print_balances_begin"),
                mock.call("process_foo__0_print_balances_end"),
                ])

            self.assertEqual(1, run_action.call_count)
            run_action.assert_has_calls([
                mock.call('fetch_balances',
                    {'checkpoint': 'end', 'log_tickers': True, 'add_usdt': True, 'add_portfolio': True},
                    {'foo': 'bar', 'tag': 'process_foo__0_print_balances_begin'}),
                ])

        self.m.reset_mock()
        with mock.patch.object(processor, "run_action") as run_action:
            step = processor.scenarios["sell_needed"][1]

            processor.process_step("foo", step, {"foo":"bar"})
            self.assertEqual(1, run_action.call_count)

        self.m.reset_mock()
        with mock.patch.object(processor, "run_action") as run_action:
            step = processor.scenarios["print_balances"][0]

            processor.process_step("foo", step, {"foo":"bar"})
            run_action.assert_has_calls([
                mock.call('fetch_balances',
                    {'log_tickers': True, 'add_usdt': True, 'add_portfolio': True},
                    {'foo': 'bar', 'tag': 'process_foo__1_print_balances_begin'}),
                ])

    def test_parse_args(self):
        processor = market.Processor(self.m)

        with mock.patch.object(processor, "method_arguments") as method_arguments:
            method_mock = mock.Mock()
            method_arguments.return_value = [
                    method_mock,
                    ["foo2", "foo"]
                    ]
            method, args = processor.parse_args("action", {"foo": "bar", "foo2": "bar"}, {"foo": "bar2", "bla": "bla"})

            self.assertEqual(method_mock, method)
            self.assertEqual({"foo": "bar2", "foo2": "bar"}, args)

        with mock.patch.object(processor, "method_arguments") as method_arguments:
            method_mock = mock.Mock()
            method_arguments.return_value = [
                    method_mock,
                    ["repartition"]
                    ]
            method, args = processor.parse_args("action", {"repartition": { "base_currency": 1 }}, {})

            self.assertEqual(1, len(args["repartition"]))
            self.assertIn("BTC", args["repartition"])

        with mock.patch.object(processor, "method_arguments") as method_arguments:
            method_mock = mock.Mock()
            method_arguments.return_value = [
                    method_mock,
                    ["repartition", "base_currency"]
                    ]
            method, args = processor.parse_args("action", {"repartition": { "base_currency": 1 }}, {"base_currency": "USDT"})

            self.assertEqual(1, len(args["repartition"]))
            self.assertIn("USDT", args["repartition"])

        with mock.patch.object(processor, "method_arguments") as method_arguments:
            method_mock = mock.Mock()
            method_arguments.return_value = [
                    method_mock,
                    ["repartition", "base_currency"]
                    ]
            method, args = processor.parse_args("action", {"repartition": { "ETH": 1 }}, {"base_currency": "USDT"})

            self.assertEqual(1, len(args["repartition"]))
            self.assertIn("ETH", args["repartition"])