4 import simplejson
as json
5 from decimal
import Decimal
as D
, ROUND_DOWN
8 from json
import JSONDecodeError
9 from simplejson
.errors
import JSONDecodeError
as SimpleJSONDecodeError
11 __all__
= ["Portfolio", "BalanceStore", "ReportStore", "TradeStore"]
14 def __init__(self
, market
, verbose_print
=True, no_http_dup
=False):
16 self
.verbose_print
= verbose_print
20 self
.redis_status
= []
22 self
.no_http_dup
= no_http_dup
25 def merge(self
, other_report
):
26 self
.logs
+= other_report
.logs
27 self
.logs
.sort(key
=lambda x
: x
["date"])
29 self
.print_logs
+= other_report
.print_logs
30 self
.print_logs
.sort(key
=lambda x
: x
[0])
32 def print_log(self
, message
):
33 now
= datetime
.datetime
.now()
34 message
= "{:%Y-%m-%d %H:%M:%S}: {}".format(now
, str(message
))
35 self
.print_logs
.append([now
, message
])
36 if self
.verbose_print
:
39 def add_log(self
, hash_
):
40 hash_
["date"] = datetime
.datetime
.now()
41 if self
.market
is not None:
42 hash_
["user_id"] = self
.market
.user_id
43 hash_
["market_id"] = self
.market
.market_id
45 hash_
["user_id"] = None
46 hash_
["market_id"] = None
47 self
.logs
.append(hash_
)
50 def add_redis_status(self
, hash_
):
51 self
.redis_status
.append(hash_
)
55 def default_json_serial(obj
):
56 if isinstance(obj
, (datetime
.datetime
, datetime
.date
)):
57 return obj
.isoformat()
61 return json
.dumps(self
.logs
, default
=self
.default_json_serial
, indent
=" ")
63 def to_json_array(self
):
64 for log
in (x
.copy() for x
in self
.logs
):
68 json
.dumps(log
, default
=self
.default_json_serial
, indent
=" ")
71 def to_json_redis(self
):
72 for log
in (x
.copy() for x
in self
.redis_status
):
75 json
.dumps(log
, default
=self
.default_json_serial
)
78 def set_verbose(self
, verbose_print
):
79 self
.verbose_print
= verbose_print
81 def log_stage(self
, stage
, **kwargs
):
84 return inspect
.getsource(element
).strip()
85 elif hasattr(element
, "as_json"):
86 return element
.as_json()
90 args
= { k: as_json(v) for k, v in kwargs.items() }
91 args_str
= ["{}={}".format(k
, v
) for k
, v
in args
.items()]
92 self
.print_log("-" * (len(stage
) + 8))
93 self
.print_log("[Stage] {} {}".format(stage
, ", ".join(args_str
)))
101 def log_balances(self
, tag
=None):
102 self
.print_log("[Balance]")
103 for currency
, balance
in self
.market
.balances
.all
.items():
104 self
.print_log("\t{}".format(balance
))
109 "balances": self
.market
.balances
.as_json()
112 self
.add_log(log
.copy())
113 self
.add_redis_status(log
)
115 def log_tickers(self
, amounts
, other_currency
,
116 compute_value
, type):
119 if callable(compute_value
):
120 compute_value
= inspect
.getsource(compute_value
).strip()
122 for currency
, amount
in amounts
.items():
123 values
[currency
] = amount
.as_json()["value"]
124 rates
[currency
] = amount
.rate
127 "compute_value": compute_value
,
128 "balance_type": type,
129 "currency": other_currency
,
132 "total": sum(amounts
.values()).as_json()["value"]
135 self
.add_log(log
.copy())
136 self
.add_redis_status(log
)
138 def log_dispatch(self
, amount
, amounts
, liquidity
, repartition
):
141 "liquidity": liquidity
,
142 "repartition_ratio": repartition
,
143 "total_amount": amount
.as_json(),
144 "repartition": { k: v.as_json()["value"] for k, v in amounts.items() }
147 def log_trades(self
, matching_and_trades
, only
):
149 for matching
, trade
in matching_and_trades
:
150 trade_json
= trade
.as_json()
151 trade_json
["skipped"] = not matching
152 trades
.append(trade_json
)
157 "debug": self
.market
.debug
,
161 def log_orders(self
, orders
, tick
=None, only
=None, compute_value
=None):
162 if callable(compute_value
):
163 compute_value
= inspect
.getsource(compute_value
).strip()
164 self
.print_log("[Orders]")
165 self
.market
.trades
.print_all_with_order(ind
="\t")
169 "compute_value": compute_value
,
171 "orders": [order
.as_json() for order
in orders
if order
is not None]
174 def log_order(self
, order
, tick
, finished
=False, update
=None,
175 new_order
=None, compute_value
=None):
176 if callable(compute_value
):
177 compute_value
= inspect
.getsource(compute_value
).strip()
179 self
.print_log("[Order] Finished {}".format(order
))
180 elif update
== "waiting":
181 self
.print_log("[Order] {}, tick {}, waiting".format(order
, tick
))
182 elif update
== "adjusting":
183 self
.print_log("[Order] {}, tick {}, cancelling and adjusting to {}".format(order
, tick
, new_order
))
184 elif update
== "market_fallback":
185 self
.print_log("[Order] {}, tick {}, fallbacking to market value".format(order
, tick
))
186 elif update
== "market_adjust":
187 self
.print_log("[Order] {}, tick {}, market value, cancelling and adjusting to {}".format(order
, tick
, new_order
))
193 "order": order
.as_json(),
194 "compute_value": compute_value
,
195 "new_order": new_order
.as_json() if new_order
is not None else None
198 def log_move_balances(self
, needed
, moving
):
200 "type": "move_balances",
201 "debug": self
.market
.debug
,
202 "needed": { k: v.as_json()["value"] if isinstance(v, portfolio.Amount) else v for k, v in needed.items() }
,
203 "moving": { k: v.as_json()["value"] if isinstance(v, portfolio.Amount) else v for k, v in moving.items() }
,
206 def log_http_request(self
, method
, url
, body
, headers
, response
):
207 if isinstance(response
, Exception):
209 "type": "http_request",
216 "error": response
.__class
__.__name
__,
217 "error_message": str(response
),
219 self
.last_http
= None
220 elif self
.no_http_dup
and \
221 self
.last_http
is not None and \
222 self
.last_http
["url"] == url
and \
223 self
.last_http
["method"] == method
and \
224 self
.last_http
["response"] == response
.text
:
226 "type": "http_request",
231 "status": response
.status_code
,
232 "duration": response
.elapsed
.total_seconds(),
234 "response_same_as": self
.last_http
["date"]
237 self
.last_http
= self
.add_log({
238 "type": "http_request",
243 "status": response
.status_code
,
244 "duration": response
.elapsed
.total_seconds(),
245 "response": response
.text
,
246 "response_same_as": None,
249 def log_error(self
, action
, message
=None, exception
=None):
250 self
.print_log("[Error] {}".format(action
))
251 if exception
is not None:
252 self
.print_log(str("\t{}: {}".format(exception
.__class
__.__name
__, exception
)))
253 if message
is not None:
254 self
.print_log("\t{}".format(message
))
259 "exception_class": exception
.__class
__.__name
__ if exception
is not None else None,
260 "exception_message": str(exception
) if exception
is not None else None,
264 def log_debug_action(self
, action
):
265 self
.print_log("[Debug] {}".format(action
))
268 "type": "debug_action",
272 def log_market(self
, args
):
275 "commit": "$Format:%H$",
280 def __init__(self
, market
):
284 def currencies(self
):
285 return self
.all
.keys()
287 def in_currency(self
, other_currency
, compute_value
="average", type="total"):
289 for currency
, balance
in self
.all
.items():
290 other_currency_amount
= getattr(balance
, type)\
291 .in_currency(other_currency
, self
.market
, compute_value
=compute_value
)
292 amounts
[currency
] = other_currency_amount
293 self
.market
.report
.log_tickers(amounts
, other_currency
,
297 def fetch_balances(self
, tag
=None):
298 all_balances
= self
.market
.ccxt
.fetch_all_balances()
299 for currency
, balance
in all_balances
.items():
300 if balance
["exchange_total"] != 0 or balance
["margin_total"] != 0 or \
301 currency
in self
.all
:
302 self
.all
[currency
] = portfolio
.Balance(currency
, balance
)
303 self
.market
.report
.log_balances(tag
=tag
)
305 def dispatch_assets(self
, amount
, liquidity
="medium", repartition
=None):
306 if repartition
is None:
307 repartition
= Portfolio
.repartition(liquidity
=liquidity
)
308 sum_ratio
= sum([v
[0] for k
, v
in repartition
.items()])
310 for currency
, (ptt
, trade_type
) in repartition
.items():
311 amounts
[currency
] = ptt
* amount
/ sum_ratio
312 if trade_type
== "short":
313 amounts
[currency
] = - amounts
[currency
]
314 self
.all
.setdefault(currency
, portfolio
.Balance(currency
, {}))
315 self
.market
.report
.log_dispatch(amount
, amounts
, liquidity
, repartition
)
319 return { k: v.as_json() for k, v in self.all.items() }
322 def __init__(self
, market
):
328 return list(filter(lambda t
: t
.pending
, self
.all
))
330 def compute_trades(self
, values_in_base
, new_repartition
, only
=None):
332 base_currency
= sum(values_in_base
.values()).currency
333 for currency
in self
.market
.balances
.currencies():
334 if currency
== base_currency
:
336 value_from
= values_in_base
.get(currency
, portfolio
.Amount(base_currency
, 0))
337 value_to
= new_repartition
.get(currency
, portfolio
.Amount(base_currency
, 0))
339 if value_from
.value
* value_to
.value
< 0:
340 computed_trades
.append(self
.trade_if_matching(
341 value_from
, portfolio
.Amount(base_currency
, 0),
342 currency
, only
=only
))
343 computed_trades
.append(self
.trade_if_matching(
344 portfolio
.Amount(base_currency
, 0), value_to
,
345 currency
, only
=only
))
347 computed_trades
.append(self
.trade_if_matching(
348 value_from
, value_to
,
349 currency
, only
=only
))
350 for matching
, trade
in computed_trades
:
352 self
.all
.append(trade
)
353 self
.market
.report
.log_trades(computed_trades
, only
)
355 def trade_if_matching(self
, value_from
, value_to
, currency
,
357 trade
= portfolio
.Trade(value_from
, value_to
, currency
,
359 matching
= only
is None or trade
.action
== only
360 return [matching
, trade
]
362 def prepare_orders(self
, only
=None, compute_value
="default"):
364 for trade
in self
.pending
:
365 if only
is None or trade
.action
== only
:
366 orders
.append(trade
.prepare_order(compute_value
=compute_value
))
367 self
.market
.report
.log_orders(orders
, only
, compute_value
)
369 def close_trades(self
):
370 for trade
in self
.all
:
373 def print_all_with_order(self
, ind
=""):
374 for trade
in self
.all
:
375 trade
.print_with_order(ind
=ind
)
377 def run_orders(self
):
378 orders
= self
.all_orders(state
="pending")
381 self
.market
.report
.log_stage("run_orders")
382 self
.market
.report
.log_orders(orders
)
384 def all_orders(self
, state
=None):
385 all_orders
= sum(map(lambda v
: v
.orders
, self
.all
), [])
389 return list(filter(lambda o
: o
.status
== state
, all_orders
))
391 def update_all_orders_status(self
):
392 for order
in self
.all_orders(state
="open"):
396 def __enter__(self
, *args
):
398 def __exit__(self
, *args
):
402 def __init__(self
, value
):
403 self
.lock
= NoopLock()
406 def start_lock(self
):
408 self
.lock
= threading
.Lock()
410 def set(self
, value
):
414 def get(self
, key
=None):
416 if key
is not None and isinstance(self
.val
, dict):
417 return self
.val
.get(key
)
421 def __getattr__(self
, key
):
423 return getattr(self
.val
, key
)
426 URL
= "https://cryptoportfolio.io/wp-content/uploads/portfolio/json/cryptoportfolio.json"
427 data
= LockedVar(None)
428 liquidities
= LockedVar({})
429 last_date
= LockedVar(None)
430 report
= LockedVar(ReportStore(None, no_http_dup
=True))
433 worker_started
= False
438 def start_worker(cls
, poll
=30):
441 cls
.worker
= threading
.Thread(name
="portfolio", daemon
=True,
442 target
=cls
.wait_for_notification
, kwargs
={"poll": poll}
)
443 cls
.worker_notify
= threading
.Event()
444 cls
.callback
= threading
.Event()
446 cls
.last_date
.start_lock()
447 cls
.liquidities
.start_lock()
448 cls
.report
.start_lock()
450 cls
.worker_tag
= "[Worker] "
451 cls
.worker_started
= True
455 def is_worker_thread(cls
):
456 if cls
.worker
is None:
460 return cls
.worker
== threading
.current_thread()
463 def wait_for_notification(cls
, poll
=30):
464 if not cls
.is_worker_thread():
465 raise RuntimeError("This method needs to be ran with the worker")
466 while cls
.worker_started
:
467 cls
.worker_notify
.wait()
468 if cls
.worker_started
:
469 cls
.worker_notify
.clear()
470 cls
.report
.print_log("[Worker] Fetching cryptoportfolio")
471 cls
.get_cryptoportfolio(refetch
=True)
476 def stop_worker(cls
):
477 cls
.worker_started
= False
478 cls
.worker_notify
.set()
481 def notify_and_wait(cls
):
483 cls
.worker_notify
.set()
487 def wait_for_recent(cls
, delta
=4, poll
=30):
488 cls
.get_cryptoportfolio()
489 while cls
.last_date
.get() is None or datetime
.datetime
.now() - cls
.last_date
.get() > datetime
.timedelta(delta
):
490 if cls
.worker
is None:
492 cls
.report
.print_log("Attempt to fetch up-to-date cryptoportfolio")
493 cls
.get_cryptoportfolio(refetch
=True)
496 def repartition(cls
, liquidity
="medium"):
497 cls
.get_cryptoportfolio()
498 liquidities
= cls
.liquidities
.get(liquidity
)
499 return liquidities
[cls
.last_date
.get()]
502 def get_cryptoportfolio(cls
, refetch
=False):
503 if cls
.data
.get() is not None and not refetch
:
505 if cls
.worker
is not None and not cls
.is_worker_thread():
506 cls
.notify_and_wait()
509 r
= requests
.get(cls
.URL
)
510 cls
.report
.log_http_request(r
.request
.method
,
511 r
.request
.url
, r
.request
.body
, r
.request
.headers
, r
)
512 except Exception as e
:
513 cls
.report
.log_error("{}get_cryptoportfolio".format(cls
.worker_tag
), exception
=e
)
516 cls
.data
.set(r
.json(parse_int
=D
, parse_float
=D
))
517 cls
.parse_cryptoportfolio()
518 except (JSONDecodeError
, SimpleJSONDecodeError
):
520 cls
.last_date
.set(None)
521 cls
.liquidities
.set({})
524 def parse_cryptoportfolio(cls
):
525 def filter_weights(weight_hash
):
526 if weight_hash
[1][0] == 0:
528 if weight_hash
[0] == "_row":
532 def clean_weights(i
):
533 def clean_weights_(h
):
534 if h
[0].endswith("s"):
535 return [h
[0][0:-1], (h
[1][i
], "short")]
537 return [h
[0], (h
[1][i
], "long")]
538 return clean_weights_
540 def parse_weights(portfolio_hash
):
541 if "weights" not in portfolio_hash
:
543 weights_hash
= portfolio_hash
["weights"]
545 for i
in range(len(weights_hash
["_row"])):
546 date
= datetime
.datetime
.strptime(weights_hash
["_row"][i
], "%Y-%m-%d")
547 weights
[date
] = dict(filter(
549 map(clean_weights(i
), weights_hash
.items())))
552 high_liquidity
= parse_weights(cls
.data
.get("portfolio_1"))
553 medium_liquidity
= parse_weights(cls
.data
.get("portfolio_2"))
555 cls
.liquidities
.set({
556 "medium": medium_liquidity
,
557 "high": high_liquidity
,
559 cls
.last_date
.set(max(
560 max(medium_liquidity
.keys(), default
=datetime
.datetime(1, 1, 1)),
561 max(high_liquidity
.keys(), default
=datetime
.datetime(1, 1, 1))