Merge branch 'redis' into dev

3 files changed, 212 insertions, 80 deletions

diff --git a/tests/test_main.py b/tests/test_main.py
index d2f8029..b650870 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -135,16 +135,16 @@ class MainTest(WebMockTestCase):
             args_mock.after = "after"
             self.assertEqual("", stdout_mock.getvalue())
 
-            main.process("config", 3, 1, args_mock, "pg_config")
+            main.process("config", 3, 1, args_mock, "pg_config", "redis_config")
 
             market_mock.from_config.assert_has_calls([
-                mock.call("config", args_mock, pg_config="pg_config", market_id=3, user_id=1),
+                mock.call("config", args_mock, pg_config="pg_config", redis_config="redis_config", market_id=3, user_id=1),
                 mock.call().process("action", before="before", after="after"),
                 ])
 
             with self.subTest(exception=True):
                 market_mock.from_config.side_effect = Exception("boo")
-                main.process(3, "config", 1, args_mock, "pg_config")
+                main.process(3, "config", 1, args_mock, "pg_config", "redis_config")
                 self.assertEqual("Exception: boo\n", stdout_mock.getvalue())
 
     def test_main(self):
@@ -159,7 +159,7 @@ class MainTest(WebMockTestCase):
                 args_mock.user = "user"
                 parse_args.return_value = args_mock
 
-                parse_config.return_value = "pg_config"
+                parse_config.return_value = ["pg_config", "redis_config"]
 
                 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
 
@@ -171,8 +171,8 @@ class MainTest(WebMockTestCase):
 
                 self.assertEqual(2, process.call_count)
                 process.assert_has_calls([
-                    mock.call("config1", 3, 1, args_mock, "pg_config"),
-                    mock.call("config2", 1, 2, args_mock, "pg_config"),
+                    mock.call("config1", 3, 1, args_mock, "pg_config", "redis_config"),
+                    mock.call("config2", 1, 2, args_mock, "pg_config", "redis_config"),
                     ])
         with self.subTest(parallel=True):
             with mock.patch("main.parse_args") as parse_args,\
@@ -187,7 +187,7 @@ class MainTest(WebMockTestCase):
                 args_mock.user = "user"
                 parse_args.return_value = args_mock
 
-                parse_config.return_value = "pg_config"
+                parse_config.return_value = ["pg_config", "redis_config"]
 
                 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
 
@@ -202,9 +202,9 @@ class MainTest(WebMockTestCase):
                 self.assertEqual(2, process.call_count)
                 process.assert_has_calls([
                     mock.call.__bool__(),
-                    mock.call("config1", 3, 1, args_mock, "pg_config"),
+                    mock.call("config1", 3, 1, args_mock, "pg_config", "redis_config"),
                     mock.call.__bool__(),
-                    mock.call("config2", 1, 2, args_mock, "pg_config"),
+                    mock.call("config2", 1, 2, args_mock, "pg_config", "redis_config"),
                     ])
         with self.subTest(quiet=True):
             with mock.patch("main.parse_args") as parse_args,\
@@ -219,7 +219,7 @@ class MainTest(WebMockTestCase):
                 args_mock.user = "user"
                 parse_args.return_value = args_mock
 
-                parse_config.return_value = "pg_config"
+                parse_config.return_value = ["pg_config", "redis_config"]
 
                 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
 
@@ -240,7 +240,7 @@ class MainTest(WebMockTestCase):
                 args_mock.user = "user"
                 parse_args.return_value = args_mock
 
-                parse_config.return_value = "pg_config"
+                parse_config.return_value = ["pg_config", "redis_config"]
 
                 fetch_markets.return_value = [[3, "config1", 1], [1, "config2", 2]]
 
@@ -259,15 +259,39 @@ class MainTest(WebMockTestCase):
                 "db_user": "user",
                 "db_password": "password",
                 "db_database": "database",
+                "redis_host": "rhost",
+                "redis_port": "rport",
+                "redis_database": "rdb",
                 "report_path": None,
                 })
 
-            result = main.parse_config(args)
+            db_config, redis_config = main.parse_config(args)
             self.assertEqual({ "host": "host", "port": "port", "user":
                 "user", "password": "password", "database": "database"
-                }, result)
+                }, db_config)
+            self.assertEqual({ "host": "rhost", "port": "rport", "db":
+                "rdb"}, redis_config)
+
             with self.assertRaises(AttributeError):
                 args.db_password
+            with self.assertRaises(AttributeError):
+                args.redis_host
+
+        with self.subTest(redis_host="socket"):
+            args = main.configargparse.Namespace(**{
+                "db_host": "host",
+                "db_port": "port",
+                "db_user": "user",
+                "db_password": "password",
+                "db_database": "database",
+                "redis_host": "/run/foo",
+                "redis_port": "rport",
+                "redis_database": "rdb",
+                "report_path": None,
+                })
+
+            db_config, redis_config = main.parse_config(args)
+            self.assertEqual({ "unix_socket_path": "/run/foo", "db": "rdb"}, redis_config)
 
         with self.subTest(report_path="present"):
             args = main.configargparse.Namespace(**{
@@ -276,6 +300,9 @@ class MainTest(WebMockTestCase):
                 "db_user": "user",
                 "db_password": "password",
                 "db_database": "database",
+                "redis_host": "rhost",
+                "redis_port": "rport",
+                "redis_database": "rdb",
                 "report_path": "report_path",
                 })
 
diff --git a/tests/test_market.py b/tests/test_market.py
index 14b23b5..e3482b8 100644
--- a/tests/test_market.py
+++ b/tests/test_market.py
@@ -530,23 +530,55 @@ class MarketTest(WebMockTestCase):
             m.store_database_report(datetime.datetime(2018, 3, 24))
             self.assertEqual(stdout_mock.getvalue(), "impossible to store report to database: Exception; Bouh\n")
 
+    @mock.patch.object(market, "redis")
+    def test_store_redis_report(self, redis):
+        connect_mock = mock.Mock()
+        redis.Redis.return_value = connect_mock
+
+        m = market.Market(self.ccxt, self.market_args(),
+                redis_config={"config": "redis_config"}, market_id=1)
+
+        with self.subTest(error=False),\
+                mock.patch.object(m, "report") as report:
+            report.to_json_redis.return_value = [
+                    ("type1", "payload1"),
+                    ("type2", "payload2"),
+                    ]
+            m.store_redis_report(datetime.datetime(2018, 3, 24))
+            connect_mock.assert_has_calls([
+                mock.call.set("/cryptoportfolio/1/2018-03-24T00:00:00/type1", "payload1", ex=31*24*60*60),
+                mock.call.set("/cryptoportfolio/1/latest/type1", "payload1"),
+                mock.call.set("/cryptoportfolio/1/2018-03-24T00:00:00/type2", "payload2", ex=31*24*60*60),
+                mock.call.set("/cryptoportfolio/1/latest/type2", "payload2"),
+                ])
+
+        connect_mock.reset_mock()
+        with self.subTest(error=True),\
+                mock.patch('sys.stdout', new_callable=StringIO) as stdout_mock:
+            redis.Redis.side_effect = Exception("Bouh")
+            m.store_redis_report(datetime.datetime(2018, 3, 24))
+            self.assertEqual(stdout_mock.getvalue(), "impossible to store report to redis: Exception; Bouh\n")
+
     def test_store_report(self):
         m = market.Market(self.ccxt, self.market_args(report_db=False), user_id=1)
         with self.subTest(file=None, pg_config=None),\
                 mock.patch.object(m, "report") as report,\
                 mock.patch.object(m, "store_database_report") as db_report,\
+                mock.patch.object(m, "store_redis_report") as redis_report,\
                 mock.patch.object(m, "store_file_report") as file_report:
             m.store_report()
             report.merge.assert_called_with(store.Portfolio.report)
 
             file_report.assert_not_called()
             db_report.assert_not_called()
+            redis_report.assert_not_called()
 
         report.reset_mock()
         m = market.Market(self.ccxt, self.market_args(report_db=False, report_path="present"), user_id=1)
         with self.subTest(file="present", pg_config=None),\
                 mock.patch.object(m, "report") as report,\
                 mock.patch.object(m, "store_file_report") as file_report,\
+                mock.patch.object(m, "store_redis_report") as redis_report,\
                 mock.patch.object(m, "store_database_report") as db_report,\
                 mock.patch.object(market.datetime, "datetime") as time_mock:
 
@@ -557,12 +589,14 @@ class MarketTest(WebMockTestCase):
             report.merge.assert_called_with(store.Portfolio.report)
             file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
             db_report.assert_not_called()
+            redis_report.assert_not_called()
 
         report.reset_mock()
         m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"), user_id=1)
         with self.subTest(file="present", pg_config=None, report_db=True),\
                 mock.patch.object(m, "report") as report,\
                 mock.patch.object(m, "store_file_report") as file_report,\
+                mock.patch.object(m, "store_redis_report") as redis_report,\
                 mock.patch.object(m, "store_database_report") as db_report,\
                 mock.patch.object(market.datetime, "datetime") as time_mock:
 
@@ -573,12 +607,14 @@ class MarketTest(WebMockTestCase):
             report.merge.assert_called_with(store.Portfolio.report)
             file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
             db_report.assert_not_called()
+            redis_report.assert_not_called()
 
         report.reset_mock()
         m = market.Market(self.ccxt, self.market_args(report_db=True), pg_config="present", user_id=1)
         with self.subTest(file=None, pg_config="present"),\
                 mock.patch.object(m, "report") as report,\
                 mock.patch.object(m, "store_file_report") as file_report,\
+                mock.patch.object(m, "store_redis_report") as redis_report,\
                 mock.patch.object(m, "store_database_report") as db_report,\
                 mock.patch.object(market.datetime, "datetime") as time_mock:
 
@@ -589,6 +625,7 @@ class MarketTest(WebMockTestCase):
             report.merge.assert_called_with(store.Portfolio.report)
             file_report.assert_not_called()
             db_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
+            redis_report.assert_not_called()
 
         report.reset_mock()
         m = market.Market(self.ccxt, self.market_args(report_db=True, report_path="present"),
@@ -596,6 +633,7 @@ class MarketTest(WebMockTestCase):
         with self.subTest(file="present", pg_config="present"),\
                 mock.patch.object(m, "report") as report,\
                 mock.patch.object(m, "store_file_report") as file_report,\
+                mock.patch.object(m, "store_redis_report") as redis_report,\
                 mock.patch.object(m, "store_database_report") as db_report,\
                 mock.patch.object(market.datetime, "datetime") as time_mock:
 
@@ -606,22 +644,54 @@ class MarketTest(WebMockTestCase):
             report.merge.assert_called_with(store.Portfolio.report)
             file_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
             db_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
+            redis_report.assert_not_called()
 
-    def test_print_orders(self):
-        m = market.Market(self.ccxt, self.market_args())
-        with mock.patch.object(m.report, "log_stage") as log_stage,\
-                mock.patch.object(m.balances, "fetch_balances") as fetch_balances,\
-                mock.patch.object(m, "prepare_trades") as prepare_trades,\
-                mock.patch.object(m.trades, "prepare_orders") as prepare_orders:
-            m.print_orders()
+        report.reset_mock()
+        m = market.Market(self.ccxt, self.market_args(report_redis=False),
+                redis_config="redis_config", user_id=1)
+        with self.subTest(redis_config="present", report_redis=False),\
+                mock.patch.object(m, "report") as report,\
+                mock.patch.object(m, "store_file_report") as file_report,\
+                mock.patch.object(m, "store_redis_report") as redis_report,\
+                mock.patch.object(m, "store_database_report") as db_report,\
+                mock.patch.object(market.datetime, "datetime") as time_mock:
 
-            log_stage.assert_called_with("print_orders")
-            fetch_balances.assert_called_with(tag="print_orders")
-            prepare_trades.assert_called_with(base_currency="BTC",
-                    compute_value="average")
-            prepare_orders.assert_called_with(compute_value="average")
+            time_mock.now.return_value = datetime.datetime(2018, 2, 25)
 
-    def test_print_balances(self):
+            m.store_report()
+            redis_report.assert_not_called()
+
+        report.reset_mock()
+        m = market.Market(self.ccxt, self.market_args(report_redis=True),
+                user_id=1)
+        with self.subTest(redis_config="absent", report_redis=True),\
+                mock.patch.object(m, "report") as report,\
+                mock.patch.object(m, "store_file_report") as file_report,\
+                mock.patch.object(m, "store_redis_report") as redis_report,\
+                mock.patch.object(m, "store_database_report") as db_report,\
+                mock.patch.object(market.datetime, "datetime") as time_mock:
+
+            time_mock.now.return_value = datetime.datetime(2018, 2, 25)
+
+            m.store_report()
+            redis_report.assert_not_called()
+
+        report.reset_mock()
+        m = market.Market(self.ccxt, self.market_args(report_redis=True),
+                redis_config="redis_config", user_id=1)
+        with self.subTest(redis_config="present", report_redis=True),\
+                mock.patch.object(m, "report") as report,\
+                mock.patch.object(m, "store_file_report") as file_report,\
+                mock.patch.object(m, "store_redis_report") as redis_report,\
+                mock.patch.object(m, "store_database_report") as db_report,\
+                mock.patch.object(market.datetime, "datetime") as time_mock:
+
+            time_mock.now.return_value = datetime.datetime(2018, 2, 25)
+
+            m.store_report()
+            redis_report.assert_called_once_with(datetime.datetime(2018, 2, 25))
+
+    def test_print_tickers(self):
         m = market.Market(self.ccxt, self.market_args())
 
         with mock.patch.object(m.balances, "in_currency") as in_currency,\
@@ -634,10 +704,8 @@ class MarketTest(WebMockTestCase):
                     "ETH": portfolio.Amount("BTC", "0.3"),
                     }
 
-            m.print_balances()
+            m.print_tickers()
 
-            log_stage.assert_called_once_with("print_balances")
-            fetch_balances.assert_called_with()
             print_log.assert_has_calls([
                 mock.call("total:"),
                 mock.call(portfolio.Amount("BTC", "0.95")),
@@ -648,8 +716,8 @@ class MarketTest(WebMockTestCase):
     @mock.patch("market.Market.store_report")
     def test_process(self, store_report, log_error, process):
         m = market.Market(self.ccxt, self.market_args())
-        with self.subTest(before=False, after=False):
-            m.process(None)
+        with self.subTest(actions=[], before=False, after=False):
+            m.process([])
 
             process.assert_not_called()
             store_report.assert_called_once()
@@ -659,9 +727,9 @@ class MarketTest(WebMockTestCase):
         log_error.reset_mock()
         store_report.reset_mock()
         with self.subTest(before=True, after=False):
-            m.process(None, before=True)
+            m.process(["foo"], before=True)
 
-            process.assert_called_once_with("sell_all", steps="before")
+            process.assert_called_once_with("foo", steps="before")
             store_report.assert_called_once()
             log_error.assert_not_called()
 
@@ -669,7 +737,7 @@ class MarketTest(WebMockTestCase):
         log_error.reset_mock()
         store_report.reset_mock()
         with self.subTest(before=False, after=True):
-            m.process(None, after=True)
+            m.process(["sell_all"], after=True)
 
             process.assert_called_once_with("sell_all", steps="after")
             store_report.assert_called_once()
@@ -678,54 +746,30 @@ class MarketTest(WebMockTestCase):
         process.reset_mock()
         log_error.reset_mock()
         store_report.reset_mock()
-        with self.subTest(before=True, after=True):
-            m.process(None, before=True, after=True)
+        with self.subTest(before=False, after=False):
+            m.process(["foo"])
 
-            process.assert_has_calls([
-                mock.call("sell_all", steps="before"),
-                mock.call("sell_all", steps="after"),
-                ])
+            process.assert_called_once_with("foo", steps="all")
             store_report.assert_called_once()
             log_error.assert_not_called()
 
         process.reset_mock()
         log_error.reset_mock()
         store_report.reset_mock()
-        with self.subTest(action="print_balances"),\
-                mock.patch.object(m, "print_balances") as print_balances:
-            m.process(["print_balances"])
+        with self.subTest(before=True, after=True):
+            m.process(["sell_all"], before=True, after=True)
 
-            process.assert_not_called()
-            log_error.assert_not_called()
+            process.assert_called_once_with("sell_all", steps="all")
             store_report.assert_called_once()
-            print_balances.assert_called_once_with()
-
-        log_error.reset_mock()
-        store_report.reset_mock()
-        with self.subTest(action="print_orders"),\
-                mock.patch.object(m, "print_orders") as print_orders,\
-                mock.patch.object(m, "print_balances") as print_balances:
-            m.process(["print_orders", "print_balances"])
-
-            process.assert_not_called()
             log_error.assert_not_called()
-            store_report.assert_called_once()
-            print_orders.assert_called_once_with()
-            print_balances.assert_called_once_with()
-
-        log_error.reset_mock()
-        store_report.reset_mock()
-        with self.subTest(action="unknown"):
-            m.process(["unknown"])
-            log_error.assert_called_once_with("market_process", message="Unknown action unknown")
-            store_report.assert_called_once()
 
+        process.reset_mock()
         log_error.reset_mock()
         store_report.reset_mock()
         with self.subTest(unhandled_exception=True):
             process.side_effect = Exception("bouh")
 
-            m.process(None, before=True)
+            m.process(["some_action"], before=True)
             log_error.assert_called_with("market_process", exception=mock.ANY)
             store_report.assert_called_once()
  
@@ -768,24 +812,39 @@ class ProcessorTest(WebMockTestCase):
         with self.assertRaises(TypeError):
             processor.select_steps(scenario, ["wait"])
 
+    def test_can_process(self):
+        processor = market.Processor(self.m)
+
+        with self.subTest(True):
+            self.assertTrue(processor.can_process("sell_all"))
+
+        with self.subTest(False):
+            self.assertFalse(processor.can_process("unknown_action"))
+
     @mock.patch("market.Processor.process_step")
     def test_process(self, process_step):
-        processor = market.Processor(self.m)
+        with self.subTest("unknown action"):
+            processor = market.Processor(self.m)
+            with self.assertRaises(TypeError):
+                processor.process("unknown_action")
 
-        processor.process("sell_all", foo="bar")
-        self.assertEqual(3, process_step.call_count)
+        with self.subTest("nominal case"):
+            processor = market.Processor(self.m)
 
-        steps = list(map(lambda x: x[1][1]["name"], process_step.mock_calls))
-        scenario_names = list(map(lambda x: x[1][0], process_step.mock_calls))
-        kwargs = list(map(lambda x: x[1][2], process_step.mock_calls))
-        self.assertEqual(["all_sell", "wait", "all_buy"], steps)
-        self.assertEqual(["sell_all", "sell_all", "sell_all"], scenario_names)
-        self.assertEqual([{"foo":"bar"}, {"foo":"bar"}, {"foo":"bar"}], kwargs)
+            processor.process("sell_all", foo="bar")
+            self.assertEqual(3, process_step.call_count)
 
-        process_step.reset_mock()
+            steps = list(map(lambda x: x[1][1]["name"], process_step.mock_calls))
+            scenario_names = list(map(lambda x: x[1][0], process_step.mock_calls))
+            kwargs = list(map(lambda x: x[1][2], process_step.mock_calls))
+            self.assertEqual(["all_sell", "wait", "all_buy"], steps)
+            self.assertEqual(["sell_all", "sell_all", "sell_all"], scenario_names)
+            self.assertEqual([{"foo":"bar"}, {"foo":"bar"}, {"foo":"bar"}], kwargs)
 
-        processor.process("sell_needed", steps=["before", "after"])
-        self.assertEqual(3, process_step.call_count)
+            process_step.reset_mock()
+
+            processor.process("sell_needed", steps=["before", "after"])
+            self.assertEqual(3, process_step.call_count)
 
     def test_method_arguments(self):
         ccxt = mock.Mock(spec=market.ccxt.poloniexE)
@@ -816,6 +875,9 @@ class ProcessorTest(WebMockTestCase):
         method, arguments = processor.method_arguments("close_trades")
         self.assertEqual(m.trades.close_trades, method)
 
+        method, arguments = processor.method_arguments("print_tickers")
+        self.assertEqual(m.print_tickers, method)
+
     def test_process_step(self):
         processor = market.Processor(self.m)
 
diff --git a/tests/test_store.py b/tests/test_store.py
index ffd2645..df113b7 100644
--- a/tests/test_store.py
+++ b/tests/test_store.py
@@ -459,6 +459,13 @@ class ReportStoreTest(WebMockTestCase):
 
             self.assertEqual({"foo": "bar", "date": mock.ANY, "user_id": None, "market_id": None}, result)
 
+    def test_add_redis_status(self):
+        report_store = market.ReportStore(self.m)
+        result = report_store.add_redis_status({"foo": "bar"})
+
+        self.assertEqual({"foo": "bar"}, result)
+        self.assertEqual(result, report_store.redis_status[0])
+
     def test_set_verbose(self):
         report_store = market.ReportStore(self.m)
         with self.subTest(verbose=True):
@@ -534,6 +541,20 @@ class ReportStoreTest(WebMockTestCase):
         self.assertEqual(("date1", "type1", '{\n  "foo": "bar",\n  "bla": "bla"\n}'), logs[0])
         self.assertEqual(("date2", "type2", '{\n  "foo": "bar",\n  "bla": "bla"\n}'), logs[1])
 
+    def test_to_json_redis(self):
+        report_store = market.ReportStore(self.m)
+        report_store.redis_status.append({
+            "type": "type1", "foo": "bar", "bla": "bla"
+            })
+        report_store.redis_status.append({
+            "type": "type2", "foo": "bar", "bla": "bla"
+            })
+        logs = list(report_store.to_json_redis())
+
+        self.assertEqual(2, len(logs))
+        self.assertEqual(("type1", '{"foo": "bar", "bla": "bla"}'), logs[0])
+        self.assertEqual(("type2", '{"foo": "bar", "bla": "bla"}'), logs[1])
+
     @mock.patch.object(market.ReportStore, "print_log")
     @mock.patch.object(market.ReportStore, "add_log")
     def test_log_stage(self, add_log, print_log):
@@ -559,7 +580,8 @@ class ReportStoreTest(WebMockTestCase):
 
     @mock.patch.object(market.ReportStore, "print_log")
     @mock.patch.object(market.ReportStore, "add_log")
-    def test_log_balances(self, add_log, print_log):
+    @mock.patch.object(market.ReportStore, "add_redis_status")
+    def test_log_balances(self, add_redis_status, add_log, print_log):
         report_store = market.ReportStore(self.m)
         self.m.balances.as_json.return_value = "json"
         self.m.balances.all = { "FOO": "bar", "BAR": "baz" }
@@ -575,10 +597,16 @@ class ReportStoreTest(WebMockTestCase):
             'balances': 'json',
             'tag': 'tag'
             })
+        add_redis_status.assert_called_once_with({
+            'type': 'balance',
+            'balances': 'json',
+            'tag': 'tag'
+            })
 
     @mock.patch.object(market.ReportStore, "print_log")
     @mock.patch.object(market.ReportStore, "add_log")
-    def test_log_tickers(self, add_log, print_log):
+    @mock.patch.object(market.ReportStore, "add_redis_status")
+    def test_log_tickers(self, add_redis_status, add_log, print_log):
         report_store = market.ReportStore(self.m)
         amounts = {
                 "BTC": portfolio.Amount("BTC", 10),
@@ -603,6 +631,21 @@ class ReportStoreTest(WebMockTestCase):
                 },
             'total': D('10.3')
             })
+        add_redis_status.assert_called_once_with({
+            'type': 'tickers',
+            'compute_value': 'default',
+            'balance_type': 'total',
+            'currency': 'BTC',
+            'balances': {
+                'BTC': D('10'),
+                'ETH': D('0.3')
+                },
+            'rates': {
+                'BTC': None,
+                'ETH': D('0.1')
+                },
+            'total': D('10.3')
+            })
 
         add_log.reset_mock()
         compute_value = lambda x: x["bid"]