Improve fix of vanishing orders

3 files changed, 94 insertions, 77 deletions

diff --git a/market.py b/market.py
index d0e6ab4..ac3aa14 100644
--- a/market.py
+++ b/market.py
@@ -181,10 +181,17 @@ class Market:
             self.report.log_stage("follow_orders_tick_{}".format(tick))
             self.report.log_orders(open_orders, tick=tick)
             for order in open_orders:
-                if order.get_status() != "open":
+                status = order.get_status()
+                if status != "open":
                     self.report.log_order(order, tick, finished=True)
                 else:
                     order.trade.update_order(order, tick)
+                if status == "error_disappeared":
+                    self.report.log_error("follow_orders",
+                            message="{} disappeared, recreating it".format(order))
+                    order.trade.prepare_order(
+                            compute_value=order.trade.tick_actions_recreate(tick))
+
         self.report.log_stage("follow_orders_end")
 
     def prepare_trades(self, base_currency="BTC", liquidity="medium",
diff --git a/portfolio.py b/portfolio.py
index 9e98c43..535aaa8 100644
--- a/portfolio.py
+++ b/portfolio.py
@@ -269,20 +269,24 @@ class Trade:
             filled_amount += order.filled_amount(in_base_currency=in_base_currency)
         return filled_amount
 
-    def update_order(self, order, tick):
-        actions = {
-                0: ["waiting", None],
-                1: ["waiting", None],
-                2: ["adjusting", lambda x, y: (x[y] + x["average"]) / 2],
-                3: ["waiting", None],
-                4: ["waiting", None],
-                5: ["adjusting", lambda x, y: (x[y]*2 + x["average"]) / 3],
-                6: ["waiting", None],
-                7: ["market_fallback", "default"],
-                }
+    tick_actions = {
+            0: ["waiting", None],
+            1: ["waiting", None],
+            2: ["adjusting", lambda x, y: (x[y] + x["average"]) / 2],
+            3: ["waiting", None],
+            4: ["waiting", None],
+            5: ["adjusting", lambda x, y: (x[y]*2 + x["average"]) / 3],
+            6: ["waiting", None],
+            7: ["market_fallback", "default"],
+            }
 
-        if tick in actions:
-            update, compute_value = actions[tick]
+    def tick_actions_recreate(self, tick, default="average"):
+        return ([default] + \
+                [ y[1] for x, y in self.tick_actions.items() if x <= tick and y[1] is not None ])[-1]
+
+    def update_order(self, order, tick):
+        if tick in self.tick_actions:
+            update, compute_value = self.tick_actions[tick]
         elif tick % 3 == 1:
             update = "market_adjust"
             compute_value = "default"
@@ -381,14 +385,6 @@ class Trade:
         self.orders.append(order)
         return order
 
-    def reopen_same_order(self, order):
-        new_order = Order(order.action, order.amount, order.rate,
-                order.base_currency, order.trade_type, self.market,
-                self, close_if_possible=order.close_if_possible)
-        self.orders.append(new_order)
-        new_order.run()
-        return new_order
-
     def as_json(self):
         return {
                 "action": self.action,
@@ -550,14 +546,11 @@ class Order:
             self.fetch()
         return self.status
 
-    def fix_disappeared_order(self):
+    def mark_disappeared_order(self):
         if self.status.startswith("closed") and \
-                len(self.mouvements) == 1 and \
-                self.mouvements[0].total_in_base == 0:
+                len(self.mouvements) > 0 and \
+                self.mouvements[-1].total_in_base == 0:
             self.status = "error_disappeared"
-            new_order = self.trade.reopen_same_order(self)
-            self.market.report.log_error("fetch",
-                    message="Order {} disappeared, recreating it as {}".format(self, new_order))
 
     def mark_finished_order(self):
         if self.status.startswith("closed") and self.market.debug:
@@ -582,7 +575,7 @@ class Order:
 
         self.fetch_mouvements()
 
-        self.fix_disappeared_order()
+        self.mark_disappeared_order()
 
         self.mark_finished_order()
         # FIXME: consider open order with dust remaining as closed
@@ -614,6 +607,7 @@ class Order:
         for mouvement_hash in mouvements:
             self.mouvements.append(Mouvement(self.amount.currency,
                 self.base_currency, mouvement_hash))
+        self.mouvements.sort(key= lambda x: x.date)
 
     def cancel(self):
         if self.market.debug:
diff --git a/test.py b/test.py
index 7b17f9e..a9a80c5 100644
--- a/test.py
+++ b/test.py
@@ -1403,6 +1403,38 @@ class MarketTest(WebMockTestCase):
                 else:
                     time_mock.assert_called_with(sleep)
 
+        with self.subTest("disappearing order"), \
+                mock.patch("market.ReportStore"):
+            all_orders.reset_mock()
+            m = market.Market(self.ccxt, self.market_args())
+
+            order_mock1 = mock.Mock()
+            order_mock2 = mock.Mock()
+            all_orders.side_effect = [
+                    [order_mock1, order_mock2],
+                    [order_mock1, order_mock2],
+
+                    [order_mock1, order_mock2],
+                    [order_mock1, order_mock2],
+
+                    []
+                    ]
+
+            order_mock1.get_status.side_effect = ["open", "closed"]
+            order_mock2.get_status.side_effect = ["open", "error_disappeared"]
+
+            order_mock1.trade = mock.Mock()
+            trade_mock = mock.Mock()
+            order_mock2.trade = trade_mock
+
+            trade_mock.tick_actions_recreate.return_value = "tick1"
+
+            m.follow_orders()
+
+            trade_mock.tick_actions_recreate.assert_called_once_with(2)
+            trade_mock.prepare_order.assert_called_once_with(compute_value="tick1")
+            m.report.log_error.assert_called_once_with("follow_orders", message=mock.ANY)
+
     @mock.patch.object(market.BalanceStore, "fetch_balances")
     def test_move_balance(self, fetch_balances):
         for debug in [True, False]:
@@ -2406,27 +2438,6 @@ class TradeTest(WebMockTestCase):
         order1.filled_amount.assert_called_with(in_base_currency=True)
         order2.filled_amount.assert_called_with(in_base_currency=True)
 
-    def test_reopen_same_order(self):
-        value_from = portfolio.Amount("BTC", "0.5")
-        value_from.linked_to = portfolio.Amount("ETH", "10.0")
-        value_to = portfolio.Amount("BTC", "1.0")
-        trade = portfolio.Trade(value_from, value_to, "ETH", self.m)
-        order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
-                D("0.1"), "BTC", "long", self.m, "trade")
-        with mock.patch("portfolio.Order.run") as run:
-            new_order = trade.reopen_same_order(order)
-            self.assertEqual("buy", new_order.action)
-            self.assertEqual(portfolio.Amount("ETH", 10), new_order.amount)
-            self.assertEqual(D("0.1"), new_order.rate)
-            self.assertEqual("BTC", new_order.base_currency)
-            self.assertEqual("long", new_order.trade_type)
-            self.assertEqual(self.m, new_order.market)
-            self.assertEqual(False, new_order.close_if_possible)
-            self.assertEqual(trade, new_order.trade)
-            run.assert_called_once()
-            self.assertEqual(1, len(trade.orders))
-            self.assertEqual(new_order, trade.orders[0])
-
     @mock.patch.object(portfolio.Computation, "compute_value")
     @mock.patch.object(portfolio.Trade, "filled_amount")
     @mock.patch.object(portfolio, "Order")
@@ -2582,6 +2593,21 @@ class TradeTest(WebMockTestCase):
                     D("125"), "FOO", "long", self.m,
                     trade, close_if_possible=False)
 
+    def test_tick_actions_recreate(self):
+        value_from = portfolio.Amount("BTC", "0.5")
+        value_from.linked_to = portfolio.Amount("ETH", "10.0")
+        value_to = portfolio.Amount("BTC", "1.0")
+        trade = portfolio.Trade(value_from, value_to, "ETH", self.m)
+
+        self.assertEqual("average", trade.tick_actions_recreate(0))
+        self.assertEqual("foo", trade.tick_actions_recreate(0, default="foo"))
+        self.assertEqual("average", trade.tick_actions_recreate(1))
+        self.assertEqual(trade.tick_actions[2][1], trade.tick_actions_recreate(2))
+        self.assertEqual(trade.tick_actions[2][1], trade.tick_actions_recreate(3))
+        self.assertEqual(trade.tick_actions[5][1], trade.tick_actions_recreate(5))
+        self.assertEqual(trade.tick_actions[5][1], trade.tick_actions_recreate(6))
+        self.assertEqual("default", trade.tick_actions_recreate(7))
+        self.assertEqual("default", trade.tick_actions_recreate(8))
 
     @mock.patch.object(portfolio.Trade, "prepare_order")
     def test_update_order(self, prepare_order):
@@ -2984,12 +3010,12 @@ class OrderTest(WebMockTestCase):
         self.m.ccxt.privatePostReturnOrderTrades.return_value = [
                 {
                     "tradeID": 42, "type": "buy", "fee": "0.0015",
-                    "date": "2017-12-30 12:00:12", "rate": "0.1",
+                    "date": "2017-12-30 13:00:12", "rate": "0.1",
                     "amount": "3", "total": "0.3"
                     },
                 {
                     "tradeID": 43, "type": "buy", "fee": "0.0015",
-                    "date": "2017-12-30 13:00:12", "rate": "0.2",
+                    "date": "2017-12-30 12:00:12", "rate": "0.2",
                     "amount": "2", "total": "0.4"
                     }
             ]
@@ -3002,8 +3028,8 @@ class OrderTest(WebMockTestCase):
 
         self.m.ccxt.privatePostReturnOrderTrades.assert_called_with({"orderNumber": 12})
         self.assertEqual(2, len(order.mouvements))
-        self.assertEqual(42, order.mouvements[0].id)
-        self.assertEqual(43, order.mouvements[1].id)
+        self.assertEqual(43, order.mouvements[0].id)
+        self.assertEqual(42, order.mouvements[1].id)
 
         self.m.ccxt.privatePostReturnOrderTrades.side_effect = portfolio.ExchangeError
         order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
@@ -3059,9 +3085,9 @@ class OrderTest(WebMockTestCase):
         self.m.report.log_debug_action.assert_called_once()
 
     @mock.patch.object(portfolio.Order, "fetch_mouvements")
-    @mock.patch.object(portfolio.Order, "fix_disappeared_order")
+    @mock.patch.object(portfolio.Order, "mark_disappeared_order")
     @mock.patch.object(portfolio.Order, "mark_finished_order")
-    def test_fetch(self, mark_finished_order, fix_disappeared_order, fetch_mouvements):
+    def test_fetch(self, mark_finished_order, mark_disappeared_order, fetch_mouvements):
         order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
                 D("0.1"), "BTC", "long", self.m, "trade")
         order.id = 45
@@ -3072,7 +3098,7 @@ class OrderTest(WebMockTestCase):
             self.m.report.log_debug_action.reset_mock()
             self.m.ccxt.fetch_order.assert_not_called()
             mark_finished_order.assert_not_called()
-            fix_disappeared_order.assert_not_called()
+            mark_disappeared_order.assert_not_called()
             fetch_mouvements.assert_not_called()
 
         with self.subTest(debug=False):
@@ -3090,7 +3116,7 @@ class OrderTest(WebMockTestCase):
             self.assertEqual(1, len(order.results))
             self.m.report.log_debug_action.assert_not_called()
             mark_finished_order.assert_called_once()
-            fix_disappeared_order.assert_called_once()
+            mark_disappeared_order.assert_called_once()
 
             mark_finished_order.reset_mock()
             with self.subTest(missing_order=True):
@@ -3101,7 +3127,7 @@ class OrderTest(WebMockTestCase):
                 self.assertEqual("closed_unknown", order.status)
                 mark_finished_order.assert_called_once()
 
-    def test_fix_disappeared_order(self):
+    def test_mark_disappeared_order(self):
         with self.subTest("Open order"):
             order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
                     D("0.1"), "BTC", "long", self.m, "trade")
@@ -3116,9 +3142,8 @@ class OrderTest(WebMockTestCase):
                 "fee":"0.00150000",
                 "date":"2018-04-02 00:09:13"
                 }))
-            with mock.patch.object(order, "trade") as trade:
-                order.fix_disappeared_order()
-                trade.reopen_same_order.assert_not_called()
+            order.mark_disappeared_order()
+            self.assertEqual("pending", order.status)
 
         with self.subTest("Non-zero amount"):
             order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
@@ -3135,10 +3160,8 @@ class OrderTest(WebMockTestCase):
                 "fee":"0.00150000",
                 "date":"2018-04-02 00:09:13"
                 }))
-            with mock.patch.object(order, "trade") as trade:
-                order.fix_disappeared_order()
-                self.assertEqual("closed", order.status)
-                trade.reopen_same_order.assert_not_called()
+            order.mark_disappeared_order()
+            self.assertEqual("closed", order.status)
 
         with self.subTest("Other mouvements"):
             order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
@@ -3165,10 +3188,8 @@ class OrderTest(WebMockTestCase):
                 "fee":"0.00150000",
                 "date":"2018-04-02 00:09:13"
                 }))
-            with mock.patch.object(order, "trade") as trade:
-                order.fix_disappeared_order()
-                self.assertEqual("closed", order.status)
-                trade.reopen_same_order.assert_not_called()
+            order.mark_disappeared_order()
+            self.assertEqual("error_disappeared", order.status)
 
         with self.subTest("Order disappeared"):
             order = portfolio.Order("buy", portfolio.Amount("ETH", 10),
@@ -3185,13 +3206,8 @@ class OrderTest(WebMockTestCase):
                 "fee":"0.00150000",
                 "date":"2018-04-02 00:09:13"
                 }))
-            with mock.patch.object(order, "trade") as trade:
-                trade.reopen_same_order.return_value = "New order"
-                order.fix_disappeared_order()
-                self.assertEqual("error_disappeared", order.status)
-                trade.reopen_same_order.assert_called_once_with(order)
-                self.m.report.log_error.assert_called_once_with('fetch',
-                        message='Order Order(buy long 10.00000000 ETH at 0.1 BTC [error_disappeared]) disappeared, recreating it as New order')
+            order.mark_disappeared_order()
+            self.assertEqual("error_disappeared", order.status)
 
     @mock.patch.object(portfolio.Order, "fetch")
     def test_get_status(self, fetch):