market.py

   1 from ccxt import ExchangeError, NotSupported, RequestTimeout, InvalidNonce
   2 import ccxt_wrapper as ccxt
   3 import time
   4 import psycopg2
   5 import redis
   6 from store import *
   7 from cachetools.func import ttl_cache
   8 from datetime import datetime
   9 import datetime
  10 from retry import retry
  11 import portfolio
  12
  13 class Market:
  14     debug = False
  15     ccxt = None
  16     report = None
  17     trades = None
  18     balances = None
  19
  20     def __init__(self, ccxt_instance, args, **kwargs):
  21         self.args = args
  22         self.debug = args.debug
  23         self.ccxt = ccxt_instance
  24         self.ccxt._market = self
  25         self.report = ReportStore(self, verbose_print=(not args.quiet))
  26         self.trades = TradeStore(self)
  27         self.balances = BalanceStore(self)
  28         self.processor = Processor(self)
  29
  30         for key in ["user_id", "market_id", "pg_config", "redis_config"]:
  31             setattr(self, key, kwargs.get(key, None))
  32
  33         self.report.log_market(self.args)
  34
  35     @classmethod
  36     def from_config(cls, config, args, **kwargs):
  37         config["apiKey"] = config.pop("key", None)
  38
  39         ccxt_instance = ccxt.poloniexE(config)
  40
  41         return cls(ccxt_instance, args, **kwargs)
  42
  43     def store_report(self):
  44         self.report.merge(Portfolio.report)
  45         date = datetime.datetime.now()
  46         if self.args.report_path is not None:
  47             self.store_file_report(date)
  48         if self.pg_config is not None and self.args.report_db:
  49             self.store_database_report(date)
  50         if self.redis_config is not None and self.args.report_redis:
  51             self.store_redis_report(date)
  52
  53     def store_file_report(self, date):
  54         try:
  55             report_file = "{}/{}_{}".format(self.args.report_path, date.isoformat(), self.user_id)
  56             with open(report_file + ".json", "w") as f:
  57                 f.write(self.report.to_json())
  58             with open(report_file + ".log", "w") as f:
  59                 f.write("\n".join(map(lambda x: x[1], self.report.print_logs)))
  60         except Exception as e:
  61             print("impossible to store report file: {}; {}".format(e.__class__.__name__, e))
  62
  63     def store_database_report(self, date):
  64         try:
  65             report_query = 'INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id;'
  66             line_query = 'INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);'
  67             connection = psycopg2.connect(**self.pg_config)
  68             cursor = connection.cursor()
  69             cursor.execute(report_query, (date, self.market_id, self.debug))
  70             report_id = cursor.fetchone()[0]
  71             for date, type_, payload in self.report.to_json_array():
  72                 cursor.execute(line_query, (date, report_id, type_, payload))
  73
  74             connection.commit()
  75             cursor.close()
  76             connection.close()
  77         except Exception as e:
  78             print("impossible to store report to database: {}; {}".format(e.__class__.__name__, e))
  79
  80     def store_redis_report(self, date):
  81         try:
  82             conn = redis.Redis(**self.redis_config)
  83             for type_, log in self.report.to_json_redis():
  84                 key = "/cryptoportfolio/{}/{}/{}".format(self.market_id, date.isoformat(), type_)
  85                 conn.set(key, log, ex=31*24*60*60)
  86                 key = "/cryptoportfolio/{}/latest/{}".format(self.market_id, type_)
  87                 conn.set(key, log)
  88             key = "/cryptoportfolio/{}/latest/date".format(self.market_id)
  89             conn.set(key, date.isoformat())
  90         except Exception as e:
  91             print("impossible to store report to redis: {}; {}".format(e.__class__.__name__, e))
  92
  93     def process(self, actions, before=False, after=False):
  94         try:
  95             for action in actions:
  96                 if bool(before) is bool(after):
  97                     self.processor.process(action, steps="all")
  98                 elif before:
  99                     self.processor.process(action, steps="before")
 100                 elif after:
 101                     self.processor.process(action, steps="after")
 102         except Exception as e:
 103             self.report.log_error("market_process", exception=e)
 104         finally:
 105             self.store_report()
 106
 107     @retry((RequestTimeout, InvalidNonce), tries=5)
 108     def move_balances(self):
 109         needed_in_margin = {}
 110         moving_to_margin = {}
 111
 112         for currency, balance in self.balances.all.items():
 113             needed_in_margin[currency] = balance.margin_in_position - balance.margin_pending_gain
 114         for trade in self.trades.pending:
 115             needed_in_margin.setdefault(trade.base_currency, 0)
 116             if trade.trade_type == "short":
 117                 needed_in_margin[trade.base_currency] -= trade.delta
 118         for currency, needed in needed_in_margin.items():
 119             current_balance = self.balances.all[currency].margin_available
 120             moving_to_margin[currency] = (needed - current_balance)
 121             delta = moving_to_margin[currency].value
 122             action = "Moving {} from exchange to margin".format(moving_to_margin[currency])
 123
 124             if self.debug and delta != 0:
 125                 self.report.log_debug_action(action)
 126                 continue
 127             try:
 128                 if delta > 0:
 129                     self.ccxt.transfer_balance(currency, delta, "exchange", "margin")
 130                 elif delta < 0:
 131                     self.ccxt.transfer_balance(currency, -delta, "margin", "exchange")
 132             except (RequestTimeout, InvalidNonce) as e:
 133                 self.report.log_error(action, message="Retrying", exception=e)
 134                 self.report.log_move_balances(needed_in_margin, moving_to_margin)
 135                 self.balances.fetch_balances()
 136                 raise e
 137         self.report.log_move_balances(needed_in_margin, moving_to_margin)
 138
 139         self.balances.fetch_balances()
 140
 141     @ttl_cache(ttl=3600)
 142     def fetch_fees(self):
 143         return self.ccxt.fetch_fees()
 144
 145     @ttl_cache(maxsize=20, ttl=5)
 146     def get_tickers(self, refresh=False):
 147         try:
 148             return self.ccxt.fetch_tickers()
 149         except NotSupported:
 150             return None
 151
 152     @ttl_cache(maxsize=20, ttl=5)
 153     def get_ticker(self, c1, c2, refresh=False):
 154         def invert(ticker):
 155             return {
 156                     "inverted": True,
 157                     "average": (1/ticker["bid"] + 1/ticker["ask"]) / 2,
 158                     "original": ticker,
 159                     }
 160         def augment_ticker(ticker):
 161             ticker.update({
 162                 "inverted": False,
 163                 "average": (ticker["bid"] + ticker["ask"] ) / 2,
 164                 })
 165             return ticker
 166
 167         tickers = self.get_tickers()
 168         if tickers is None:
 169             try:
 170                 ticker = augment_ticker(self.ccxt.fetch_ticker("{}/{}".format(c1, c2)))
 171             except ExchangeError:
 172                 try:
 173                     ticker = invert(augment_ticker(self.ccxt.fetch_ticker("{}/{}".format(c2, c1))))
 174                 except ExchangeError:
 175                     ticker = None
 176         else:
 177             if "{}/{}".format(c1, c2) in tickers:
 178                 ticker = augment_ticker(tickers["{}/{}".format(c1, c2)])
 179             elif "{}/{}".format(c2, c1) in tickers:
 180                 ticker = invert(augment_ticker(tickers["{}/{}".format(c2, c1)]))
 181             else:
 182                 ticker = None
 183         return ticker
 184
 185     def follow_orders(self, sleep=None):
 186         if sleep is None:
 187             sleep = 7 if self.debug else 30
 188             if self.debug:
 189                 self.report.log_debug_action("Set follow_orders tick to {}s".format(sleep))
 190         tick = 0
 191         self.report.log_stage("follow_orders_begin")
 192         while len(self.trades.all_orders(state="open")) > 0:
 193             time.sleep(sleep)
 194             tick += 1
 195             open_orders = self.trades.all_orders(state="open")
 196             self.report.log_stage("follow_orders_tick_{}".format(tick))
 197             self.report.log_orders(open_orders, tick=tick)
 198             for order in open_orders:
 199                 status = order.get_status()
 200                 if status != "open":
 201                     self.report.log_order(order, tick, finished=True)
 202                 else:
 203                     order.trade.update_order(order, tick)
 204                 if status == "error_disappeared":
 205                     self.report.log_error("follow_orders",
 206                             message="{} disappeared, recreating it".format(order))
 207                     order.trade.prepare_order(
 208                             compute_value=order.trade.tick_actions_recreate(tick))
 209
 210         self.report.log_stage("follow_orders_end")
 211
 212     def prepare_trades(self, base_currency="BTC", liquidity="medium",
 213             compute_value="average", repartition=None, only=None):
 214
 215         self.report.log_stage("prepare_trades",
 216                 base_currency=base_currency, liquidity=liquidity,
 217                 compute_value=compute_value, only=only,
 218                 repartition=repartition)
 219
 220         values_in_base = self.balances.in_currency(base_currency,
 221                 compute_value=compute_value)
 222         total_base_value = sum(values_in_base.values())
 223         new_repartition = self.balances.dispatch_assets(total_base_value,
 224                 liquidity=liquidity, repartition=repartition)
 225         self.trades.compute_trades(values_in_base, new_repartition, only=only)
 226
 227     def print_tickers(self, base_currency="BTC"):
 228         if base_currency is not None:
 229             self.report.print_log("total:")
 230             self.report.print_log(sum(self.balances.in_currency(base_currency).values()))
 231
 232 class Processor:
 233     scenarios = {
 234             "wait_for_cryptoportfolio": [
 235                 {
 236                     "name": "wait",
 237                     "number": 1,
 238                     "before": False,
 239                     "after": True,
 240                     "wait_for_recent": {},
 241                     },
 242                 ],
 243             "print_balances": [
 244                 {
 245                     "name": "print_balances",
 246                     "number": 1,
 247                     "fetch_balances": ["begin"],
 248                     "print_tickers": { "base_currency": "BTC" },
 249                     }
 250                 ],
 251             "print_orders": [
 252                 {
 253                     "name": "wait",
 254                     "number": 1,
 255                     "before": True,
 256                     "after": False,
 257                     "wait_for_recent": {},
 258                     },
 259                 {
 260                     "name": "make_orders",
 261                     "number": 2,
 262                     "before": False,
 263                     "after": True,
 264                     "fetch_balances": ["begin"],
 265                     "prepare_trades": { "compute_value": "average" },
 266                     "prepare_orders": { "compute_value": "average" },
 267                     },
 268                 ],
 269             "sell_needed": [
 270                 {
 271                     "name": "wait",
 272                     "number": 0,
 273                     "before": False,
 274                     "after": True,
 275                     "wait_for_recent": {},
 276                     },
 277                 {
 278                     "name": "sell",
 279                     "number": 1,
 280                     "before": False,
 281                     "after": True,
 282                     "fetch_balances": ["begin", "end"],
 283                     "prepare_trades": {},
 284                     "prepare_orders": { "only": "dispose", "compute_value": "average" },
 285                     "run_orders": {},
 286                     "follow_orders": {},
 287                     "close_trades": {},
 288                     },
 289                 {
 290                     "name": "buy",
 291                     "number": 2,
 292                     "before": False,
 293                     "after": True,
 294                     "fetch_balances": ["begin", "end"],
 295                     "prepare_trades": { "only": "acquire" },
 296                     "prepare_orders": { "only": "acquire", "compute_value": "average" },
 297                     "move_balances": {},
 298                     "run_orders": {},
 299                     "follow_orders": {},
 300                     "close_trades": {},
 301                     },
 302                 ],
 303             "sell_all": [
 304                 {
 305                     "name": "all_sell",
 306                     "number": 1,
 307                     "before": True,
 308                     "after": False,
 309                     "fetch_balances": ["begin", "end"],
 310                     "prepare_trades": { "repartition": { "base_currency": (1, "long") } },
 311                     "prepare_orders": { "compute_value": "average" },
 312                     "run_orders": {},
 313                     "follow_orders": {},
 314                     "close_trades": {},
 315                     },
 316                 {
 317                     "name": "wait",
 318                     "number": 2,
 319                     "before": False,
 320                     "after": True,
 321                     "wait_for_recent": {},
 322                     },
 323                 {
 324                     "name": "all_buy",
 325                     "number": 3,
 326                     "before": False,
 327                     "after": True,
 328                     "fetch_balances": ["begin", "end"],
 329                     "prepare_trades": {},
 330                     "prepare_orders": { "compute_value": "average" },
 331                     "move_balances": {},
 332                     "run_orders": {},
 333                     "follow_orders": {},
 334                     "close_trades": {},
 335                     },
 336                 ]
 337             }
 338
 339     ordered_actions = [
 340             "wait_for_recent", "prepare_trades", "prepare_orders",
 341             "move_balances", "run_orders", "follow_orders",
 342             "close_trades", "print_tickers"]
 343
 344     def __init__(self, market):
 345         self.market = market
 346
 347     def select_steps(self, scenario, step):
 348         if step == "all":
 349             return scenario
 350         elif step == "before" or step == "after":
 351             return list(filter(lambda x: x.get(step, False), scenario))
 352         elif type(step) == int:
 353             return [scenario[step-1]]
 354         elif type(step) == str:
 355             return list(filter(lambda x: x["name"] == step, scenario))
 356         else:
 357             raise TypeError("Unknown step {}".format(step))
 358
 359     def can_process(self, scenario_name):
 360         return scenario_name in self.scenarios
 361
 362     def process(self, scenario_name, steps="all", **kwargs):
 363         if not self.can_process(scenario_name):
 364             raise TypeError("Unknown scenario {}".format(scenario_name))
 365         scenario = self.scenarios[scenario_name]
 366         selected_steps = []
 367
 368         if type(steps) == str or type(steps) == int:
 369             selected_steps += self.select_steps(scenario, steps)
 370         else:
 371             for step in steps:
 372                 selected_steps += self.select_steps(scenario, step)
 373         for step in selected_steps:
 374             self.process_step(scenario_name, step, kwargs)
 375
 376     def process_step(self, scenario_name, step, kwargs):
 377         process_name = "process_{}__{}_{}".format(scenario_name, step["number"], step["name"])
 378         self.market.report.log_stage("{}_begin".format(process_name))
 379         if "begin" in step.get("fetch_balances", []):
 380             self.market.balances.fetch_balances(tag="{}_begin".format(process_name))
 381
 382         for action in self.ordered_actions:
 383             if action in step:
 384                 self.run_action(action, step[action], kwargs)
 385
 386         if "end" in step.get("fetch_balances", []):
 387             self.market.balances.fetch_balances(tag="{}_end".format(process_name))
 388         self.market.report.log_stage("{}_end".format(process_name))
 389
 390     def method_arguments(self, action):
 391         import inspect
 392
 393         if action == "wait_for_recent":
 394             method = Portfolio.wait_for_recent
 395         elif action == "prepare_trades":
 396             method = self.market.prepare_trades
 397         elif action == "prepare_orders":
 398             method = self.market.trades.prepare_orders
 399         elif action == "move_balances":
 400             method = self.market.move_balances
 401         elif action == "run_orders":
 402             method = self.market.trades.run_orders
 403         elif action == "follow_orders":
 404             method = self.market.follow_orders
 405         elif action == "close_trades":
 406             method = self.market.trades.close_trades
 407         elif action == "print_tickers":
 408             method = self.market.print_tickers
 409
 410         signature = inspect.getfullargspec(method)
 411         defaults = signature.defaults or []
 412         kwargs = signature.args[-len(defaults):]
 413
 414         return [method, kwargs]
 415
 416     def parse_args(self, action, default_args, kwargs):
 417         method, allowed_arguments = self.method_arguments(action)
 418         args = {k: v for k, v in {**default_args, **kwargs}.items() if k in allowed_arguments }
 419
 420         if "repartition" in args and "base_currency" in args["repartition"]:
 421             r = args["repartition"]
 422             r[args.get("base_currency", "BTC")] = r.pop("base_currency")
 423
 424         return method, args
 425
 426     def run_action(self, action, default_args, kwargs):
 427         method, args = self.parse_args(action, default_args, kwargs)
 428
 429         method(**args)