market.py

   1 from ccxt import ExchangeError, NotSupported, RequestTimeout, InvalidNonce
   2 import ccxt_wrapper as ccxt
   3 import time
   4 import dbs
   5 from store import *
   6 from cachetools.func import ttl_cache
   7 from datetime import datetime
   8 import datetime
   9 from retry import retry
  10 import portfolio
  11
  12 class Market:
  13     debug = False
  14     ccxt = None
  15     report = None
  16     trades = None
  17     balances = None
  18
  19     def __init__(self, ccxt_instance, args, **kwargs):
  20         self.args = args
  21         self.debug = args.debug
  22         self.ccxt = ccxt_instance
  23         self.ccxt._market = self
  24         self.report = ReportStore(self, verbose_print=(not args.quiet))
  25         self.trades = TradeStore(self)
  26         self.balances = BalanceStore(self)
  27         self.processor = Processor(self)
  28
  29         for key in ["user_id", "market_id"]:
  30             setattr(self, key, kwargs.get(key, None))
  31
  32         self.report.log_market(self.args)
  33
  34     @classmethod
  35     def from_config(cls, config, args, **kwargs):
  36         config["apiKey"] = config.pop("key", None)
  37
  38         ccxt_instance = ccxt.poloniexE(config)
  39
  40         return cls(ccxt_instance, args, **kwargs)
  41
  42     def store_report(self):
  43         self.report.merge(Portfolio.report)
  44         date = datetime.datetime.now()
  45         if self.args.report_path is not None:
  46             self.store_file_report(date)
  47         if dbs.psql_connected() and self.args.report_db:
  48             self.store_database_report(date)
  49         if dbs.redis_connected() and self.args.report_redis:
  50             self.store_redis_report(date)
  51
  52     def store_file_report(self, date):
  53         try:
  54             report_file = "{}/{}_{}".format(self.args.report_path, date.isoformat(), self.user_id)
  55             with open(report_file + ".json", "w") as f:
  56                 f.write(self.report.to_json())
  57             with open(report_file + ".log", "w") as f:
  58                 f.write("\n".join(map(lambda x: x[1], self.report.print_logs)))
  59         except Exception as e:
  60             print("impossible to store report file: {}; {}".format(e.__class__.__name__, e))
  61
  62     def store_database_report(self, date):
  63         try:
  64             report_query = 'INSERT INTO reports("date", "market_config_id", "debug") VALUES (%s, %s, %s) RETURNING id;'
  65             line_query = 'INSERT INTO report_lines("date", "report_id", "type", "payload") VALUES (%s, %s, %s, %s);'
  66             cursor = dbs.psql.cursor()
  67             cursor.execute(report_query, (date, self.market_id, self.debug))
  68             report_id = cursor.fetchone()[0]
  69             for date, type_, payload in self.report.to_json_array():
  70                 cursor.execute(line_query, (date, report_id, type_, payload))
  71
  72             dbs.psql.commit()
  73             cursor.close()
  74         except Exception as e:
  75             print("impossible to store report to database: {}; {}".format(e.__class__.__name__, e))
  76
  77     def store_redis_report(self, date):
  78         try:
  79             for type_, log in self.report.to_json_redis():
  80                 key = "/cryptoportfolio/{}/{}/{}".format(self.market_id, date.isoformat(), type_)
  81                 dbs.redis.set(key, log, ex=31*24*60*60)
  82                 key = "/cryptoportfolio/{}/latest/{}".format(self.market_id, type_)
  83                 dbs.redis.set(key, log)
  84             key = "/cryptoportfolio/{}/latest/date".format(self.market_id)
  85             dbs.redis.set(key, date.isoformat())
  86         except Exception as e:
  87             print("impossible to store report to redis: {}; {}".format(e.__class__.__name__, e))
  88
  89     def process(self, actions, before=False, after=False):
  90         try:
  91             for action in actions:
  92                 if bool(before) is bool(after):
  93                     self.processor.process(action, steps="all")
  94                 elif before:
  95                     self.processor.process(action, steps="before")
  96                 elif after:
  97                     self.processor.process(action, steps="after")
  98         except Exception as e:
  99             import traceback
 100             self.report.log_error("market_process", exception=e, message=traceback.format_exc())
 101         finally:
 102             self.store_report()
 103
 104     @retry((RequestTimeout, InvalidNonce), tries=5)
 105     def move_balances(self):
 106         needed_in_margin = {}
 107         moving_to_margin = {}
 108
 109         for currency, balance in self.balances.all.items():
 110             needed_in_margin[currency] = balance.margin_in_position - balance.margin_pending_gain
 111         for trade in self.trades.pending:
 112             needed_in_margin.setdefault(trade.base_currency, 0)
 113             if trade.trade_type == "short":
 114                 needed_in_margin[trade.base_currency] -= trade.delta
 115         for currency, needed in needed_in_margin.items():
 116             current_balance = self.balances.all[currency].margin_available
 117             moving_to_margin[currency] = (needed - current_balance)
 118             delta = moving_to_margin[currency].value
 119             action = "Moving {} from exchange to margin".format(moving_to_margin[currency])
 120
 121             if self.debug and delta != 0:
 122                 self.report.log_debug_action(action)
 123                 continue
 124             try:
 125                 if delta > 0:
 126                     self.ccxt.transfer_balance(currency, delta, "exchange", "margin")
 127                 elif delta < 0:
 128                     self.ccxt.transfer_balance(currency, -delta, "margin", "exchange")
 129             except (RequestTimeout, InvalidNonce) as e:
 130                 self.report.log_error(action, message="Retrying", exception=e)
 131                 self.report.log_move_balances(needed_in_margin, moving_to_margin)
 132                 self.balances.fetch_balances()
 133                 raise e
 134         self.report.log_move_balances(needed_in_margin, moving_to_margin)
 135
 136         self.balances.fetch_balances()
 137
 138     @ttl_cache(ttl=3600)
 139     def fetch_fees(self):
 140         return self.ccxt.fetch_fees()
 141
 142     @ttl_cache(maxsize=20, ttl=5)
 143     def get_tickers(self, refresh=False):
 144         try:
 145             return self.ccxt.fetch_tickers()
 146         except NotSupported:
 147             return None
 148
 149     @ttl_cache(maxsize=20, ttl=5)
 150     def get_ticker(self, c1, c2, refresh=False):
 151         def invert(ticker):
 152             return {
 153                     "inverted": True,
 154                     "average": (1/ticker["bid"] + 1/ticker["ask"]) / 2,
 155                     "original": ticker,
 156                     }
 157         def augment_ticker(ticker):
 158             ticker.update({
 159                 "inverted": False,
 160                 "average": (ticker["bid"] + ticker["ask"] ) / 2,
 161                 })
 162             return ticker
 163
 164         tickers = self.get_tickers()
 165         if tickers is None:
 166             try:
 167                 ticker = augment_ticker(self.ccxt.fetch_ticker("{}/{}".format(c1, c2)))
 168             except ExchangeError:
 169                 try:
 170                     ticker = invert(augment_ticker(self.ccxt.fetch_ticker("{}/{}".format(c2, c1))))
 171                 except ExchangeError:
 172                     ticker = None
 173         else:
 174             if "{}/{}".format(c1, c2) in tickers:
 175                 ticker = augment_ticker(tickers["{}/{}".format(c1, c2)])
 176             elif "{}/{}".format(c2, c1) in tickers:
 177                 ticker = invert(augment_ticker(tickers["{}/{}".format(c2, c1)]))
 178             else:
 179                 ticker = None
 180         return ticker
 181
 182     def follow_orders(self, sleep=None):
 183         if sleep is None:
 184             sleep = 7 if self.debug else 30
 185             if self.debug:
 186                 self.report.log_debug_action("Set follow_orders tick to {}s".format(sleep))
 187         tick = 0
 188         self.report.log_stage("follow_orders_begin")
 189         while len(self.trades.all_orders(state="open")) > 0:
 190             time.sleep(sleep)
 191             tick += 1
 192             open_orders = self.trades.all_orders(state="open")
 193             self.report.log_stage("follow_orders_tick_{}".format(tick))
 194             self.report.log_orders(open_orders, tick=tick)
 195             for order in open_orders:
 196                 status = order.get_status()
 197                 if status != "open":
 198                     self.report.log_order(order, tick, finished=True)
 199                 else:
 200                     order.trade.update_order(order, tick)
 201                 if status == "error_disappeared":
 202                     self.report.log_error("follow_orders",
 203                             message="{} disappeared, recreating it".format(order))
 204                     new_order = order.trade.prepare_order(
 205                             compute_value=order.trade.tick_actions_recreate(tick))
 206                     new_order.run()
 207                     self.report.log_order(order, tick, new_order=new_order)
 208
 209         self.report.log_stage("follow_orders_end")
 210
 211     def prepare_trades(self, base_currency="BTC", liquidity="medium",
 212             compute_value="average", repartition=None, only=None,
 213             available_balance_only=False):
 214
 215         self.report.log_stage("prepare_trades",
 216                 base_currency=base_currency, liquidity=liquidity,
 217                 compute_value=compute_value, only=only,
 218                 repartition=repartition, available_balance_only=available_balance_only)
 219
 220         values_in_base = self.balances.in_currency(base_currency,
 221                 compute_value=compute_value)
 222         if available_balance_only:
 223             balance = self.balances.all.get(base_currency)
 224             if balance is None:
 225                 total_base_value = portfolio.Amount(base_currency, 0)
 226             else:
 227                 total_base_value = balance.exchange_free + balance.margin_available
 228         else:
 229             total_base_value = sum(values_in_base.values())
 230         new_repartition = self.balances.dispatch_assets(total_base_value,
 231                 liquidity=liquidity, repartition=repartition)
 232         if available_balance_only:
 233             for currency, amount in values_in_base.items():
 234                 if currency != base_currency:
 235                     new_repartition.setdefault(currency, portfolio.Amount(base_currency, 0))
 236                     new_repartition[currency] += amount
 237
 238         self.trades.compute_trades(values_in_base, new_repartition, only=only)
 239
 240     def print_tickers(self, base_currency="BTC"):
 241         if base_currency is not None:
 242             self.report.print_log("total:")
 243             self.report.print_log(sum(self.balances.in_currency(base_currency).values()))
 244
 245 class Processor:
 246     scenarios = {
 247             "wait_for_cryptoportfolio": [
 248                 {
 249                     "name": "wait",
 250                     "number": 1,
 251                     "before": False,
 252                     "after": True,
 253                     "wait_for_recent": {},
 254                     },
 255                 ],
 256             "print_balances": [
 257                 {
 258                     "name": "print_balances",
 259                     "number": 1,
 260                     "fetch_balances_begin": {
 261                         "log_tickers": True,
 262                         "add_usdt": True,
 263                         "add_portfolio": True
 264                         },
 265                     "print_tickers": { "base_currency": "BTC" },
 266                     }
 267                 ],
 268             "print_orders": [
 269                 {
 270                     "name": "wait",
 271                     "number": 1,
 272                     "before": True,
 273                     "after": False,
 274                     "wait_for_recent": {},
 275                     },
 276                 {
 277                     "name": "make_orders",
 278                     "number": 2,
 279                     "before": False,
 280                     "after": True,
 281                     "fetch_balances_begin": {},
 282                     "prepare_trades": { "compute_value": "average" },
 283                     "prepare_orders": { "compute_value": "average" },
 284                     },
 285                 ],
 286             "sell_needed": [
 287                 {
 288                     "name": "print_balances",
 289                     "number": 0,
 290                     "before": True,
 291                     "after": False,
 292                     "fetch_balances_begin": {
 293                         "checkpoint": "end",
 294                         "log_tickers": True,
 295                         "add_usdt": True,
 296                         "add_portfolio": True
 297                         },
 298                     },
 299                 {
 300                     "name": "wait",
 301                     "number": 1,
 302                     "before": False,
 303                     "after": True,
 304                     "wait_for_recent": {},
 305                     },
 306                 {
 307                     "name": "sell",
 308                     "number": 2,
 309                     "before": False,
 310                     "after": True,
 311                     "fetch_balances_begin": {},
 312                     "fetch_balances_end": {},
 313                     "prepare_trades": {},
 314                     "prepare_orders": { "only": "dispose", "compute_value": "average" },
 315                     "run_orders": {},
 316                     "follow_orders": {},
 317                     "close_trades": {},
 318                     },
 319                 {
 320                     "name": "buy",
 321                     "number": 3,
 322                     "before": False,
 323                     "after": True,
 324                     "fetch_balances_begin": {},
 325                     "fetch_balances_end": {
 326                         "checkpoint": "begin",
 327                         "add_usdt": True,
 328                         "log_tickers": True
 329                         },
 330                     "prepare_trades": { "only": "acquire", "available_balance_only": True },
 331                     "prepare_orders": { "only": "acquire", "compute_value": "average" },
 332                     "move_balances": {},
 333                     "run_orders": {},
 334                     "follow_orders": {},
 335                     "close_trades": {},
 336                     },
 337                 ],
 338             "sell_all": [
 339                 {
 340                     "name": "all_sell",
 341                     "number": 1,
 342                     "before": True,
 343                     "after": False,
 344                     "fetch_balances_begin": {
 345                         "checkpoint": "end",
 346                         "log_tickers": True,
 347                         "add_usdt": True,
 348                         "add_portfolio": True
 349                         },
 350                     "fetch_balances_end": {},
 351                     "prepare_trades": { "repartition": { "base_currency": (1, "long") } },
 352                     "prepare_orders": { "compute_value": "average" },
 353                     "run_orders": {},
 354                     "follow_orders": {},
 355                     "close_trades": {},
 356                     },
 357                 {
 358                     "name": "wait",
 359                     "number": 2,
 360                     "before": False,
 361                     "after": True,
 362                     "wait_for_recent": {},
 363                     },
 364                 {
 365                     "name": "all_buy",
 366                     "number": 3,
 367                     "before": False,
 368                     "after": True,
 369                     "fetch_balances_begin": {},
 370                     "fetch_balances_end": {
 371                         "checkpoint": "begin",
 372                         "add_usdt": True,
 373                         "log_tickers": True
 374                         },
 375                     "prepare_trades": { "available_balance_only": True },
 376                     "prepare_orders": { "compute_value": "average" },
 377                     "move_balances": {},
 378                     "run_orders": {},
 379                     "follow_orders": {},
 380                     "close_trades": {},
 381                     },
 382                 ]
 383             }
 384
 385     ordered_actions = [
 386             "wait_for_recent", "prepare_trades", "prepare_orders",
 387             "move_balances", "run_orders", "follow_orders",
 388             "close_trades", "print_tickers"]
 389
 390     def __init__(self, market):
 391         self.market = market
 392
 393     def select_steps(self, scenario, step):
 394         if step == "all":
 395             return scenario
 396         elif step == "before" or step == "after":
 397             return list(filter(lambda x: x.get(step, False), scenario))
 398         elif type(step) == int:
 399             return [scenario[step-1]]
 400         elif type(step) == str:
 401             return list(filter(lambda x: x["name"] == step, scenario))
 402         else:
 403             raise TypeError("Unknown step {}".format(step))
 404
 405     def can_process(self, scenario_name):
 406         return scenario_name in self.scenarios
 407
 408     def process(self, scenario_name, steps="all", **kwargs):
 409         if not self.can_process(scenario_name):
 410             raise TypeError("Unknown scenario {}".format(scenario_name))
 411         scenario = self.scenarios[scenario_name]
 412         selected_steps = []
 413
 414         if type(steps) == str or type(steps) == int:
 415             selected_steps += self.select_steps(scenario, steps)
 416         else:
 417             for step in steps:
 418                 selected_steps += self.select_steps(scenario, step)
 419         for step in selected_steps:
 420             self.process_step(scenario_name, step, kwargs)
 421
 422     def process_step(self, scenario_name, step, kwargs):
 423         process_name = "process_{}__{}_{}".format(scenario_name, step["number"], step["name"])
 424         self.market.report.log_stage("{}_begin".format(process_name))
 425
 426         if "fetch_balances_begin" in step:
 427             self.market.balances.fetch_balances(tag="{}_begin".format(process_name),
 428                     **step["fetch_balances_begin"])
 429
 430         for action in self.ordered_actions:
 431             if action in step:
 432                 self.run_action(action, step[action], kwargs)
 433
 434         if "fetch_balances_end" in step:
 435             self.market.balances.fetch_balances(tag="{}_end".format(process_name),
 436                     **step["fetch_balances_end"])
 437
 438         self.market.report.log_stage("{}_end".format(process_name))
 439
 440     def method_arguments(self, action):
 441         import inspect
 442
 443         if action == "wait_for_recent":
 444             method = Portfolio.wait_for_recent
 445         elif action == "prepare_trades":
 446             method = self.market.prepare_trades
 447         elif action == "prepare_orders":
 448             method = self.market.trades.prepare_orders
 449         elif action == "move_balances":
 450             method = self.market.move_balances
 451         elif action == "run_orders":
 452             method = self.market.trades.run_orders
 453         elif action == "follow_orders":
 454             method = self.market.follow_orders
 455         elif action == "close_trades":
 456             method = self.market.trades.close_trades
 457         elif action == "print_tickers":
 458             method = self.market.print_tickers
 459
 460         signature = inspect.getfullargspec(method)
 461         defaults = signature.defaults or []
 462         kwargs = signature.args[-len(defaults):]
 463
 464         return [method, kwargs]
 465
 466     def parse_args(self, action, default_args, kwargs):
 467         method, allowed_arguments = self.method_arguments(action)
 468         args = {k: v for k, v in {**default_args, **kwargs}.items() if k in allowed_arguments }
 469
 470         if "repartition" in args and "base_currency" in args["repartition"]:
 471             r = args["repartition"]
 472             r[args.get("base_currency", "BTC")] = r.pop("base_currency")
 473
 474         return method, args
 475
 476     def run_action(self, action, default_args, kwargs):
 477         method, args = self.parse_args(action, default_args, kwargs)
 478
 479         method(**args)