""" Hyperliquid <-> DB state reconciliation (P1.2). Runs every RECONCILE_INTERVAL_SECONDS. For each subscription with a saved HL API key (and NOT in paper mode), fetches actual open positions from HL and compares them to BotTrade rows with closed_at IS NULL. Two drift cases: 1. DB has open trade, HL has no matching position → User closed manually on HL UI, or HL liquidated, or our open_position call failed silently after writing the row. Either way the row is stale. Action: mark the DB row closed with pnl_usd=NULL, reason="closed_externally". 2. HL has open position, DB has no matching open trade → A trade exists on HL that we didn't open (legacy position, or stale wallet shared across systems). We can't safely manage it — log a warning + WS broadcast for the user to handle manually. NOTE: this does NOT close stray HL positions. Auto-closing positions we didn't open is exactly the kind of "helpful" behaviour that costs users money. We report, the human decides. Cost note: HL's user_state endpoint is cheap. 1 call per active subscriber per RECONCILE_INTERVAL. With 10 subscribers at 60s interval that's 600 calls/hr, well inside HL's free tier. """ from __future__ import annotations import asyncio import logging from datetime import datetime, timezone from typing import Optional from sqlalchemy import select, update from app.config import settings from app.database import AsyncSessionLocal from app.models import BotTrade, Subscription from app.services.crypto import decrypt_api_key from app.services.hyperliquid import HyperliquidTrader logger = logging.getLogger(__name__) RECONCILE_INTERVAL_SECONDS = 60 # ─── Single-wallet reconcile ──────────────────────────────────────────────── async def _reconcile_wallet(sub: Subscription) -> dict: """Compare HL state vs DB state for one subscription. Returns a small summary dict for logging / WS broadcast. Never raises — network / HL errors are logged and counted but don't halt the cycle. """ summary = { "wallet": sub.wallet_address, "orphan_hl": [], "orphan_db": [], "marked_closed": 0, "error": None, } # Paper-mode subs aren't on HL at all — nothing to reconcile. if sub.paper_mode: return summary try: api_key = decrypt_api_key(sub.hl_api_key) except Exception as exc: summary["error"] = f"decrypt_failed: {exc}" return summary try: trader = HyperliquidTrader( api_private_key=api_key, account_address=sub.wallet_address, leverage=sub.leverage, mainnet=settings.hl_mainnet, ) hl_positions = await trader.get_open_positions() except Exception as exc: summary["error"] = f"hl_query_failed: {exc}" return summary hl_assets_open = {p["coin"]: p for p in hl_positions} async with AsyncSessionLocal() as db: # All DB rows we believe are still open AND not released for this # wallet. A released trade left the HL position open but handed # control back to the user — reconciler must not touch it. rows = await db.execute( select(BotTrade).where( BotTrade.wallet_address == sub.wallet_address, BotTrade.closed_at.is_(None), BotTrade.released_at.is_(None), ) ) db_open_trades = rows.scalars().all() db_assets_open = {t.asset: t for t in db_open_trades} # Case 1: DB open trade, HL has nothing. Stale row → mark closed. now_naive = datetime.now(timezone.utc).replace(tzinfo=None) for asset, trade in db_assets_open.items(): # Paper trades never have HL positions; they're managed by the # normal close path. Skip here. if trade.hl_order_id == "paper": continue if asset not in hl_assets_open: # Preserve any de-risk PnL ALREADY banked on this trade — the # open remainder's PnL is unknown (closed externally) but the # banked slice is real money on HL; setting pnl_usd=None would # silently undercount realized PnL in analytics / KPIs. banked = trade.realized_partial_pnl_usd or 0.0 preserved_pnl = round(banked, 2) if banked else None # Atomic claim — same conditional UPDATE pattern as close_and_finalize. claimed = await db.execute( update(BotTrade) .where(BotTrade.id == trade.id) .where(BotTrade.closed_at.is_(None)) .values(closed_at=now_naive, pnl_usd=preserved_pnl, remaining_fraction=0.0) ) if claimed.rowcount > 0: summary["marked_closed"] += 1 summary["orphan_db"].append({"asset": asset, "trade_id": trade.id}) # Stop the TP/SL watcher for this trade from app.services.tp_sl_monitor import unregister unregister(trade.id) logger.warning("Reconcile: trade %d (%s %s) closed externally — marked.", trade.id, trade.side, asset) await db.commit() # Case 2: HL has position we don't know about. Report only — don't auto-trade. for asset, hl_pos in hl_assets_open.items(): if asset not in db_assets_open: summary["orphan_hl"].append({ "asset": asset, "szi": hl_pos["szi"], "entry": hl_pos["entry_px"], "uPnL": hl_pos["unrealized_pnl"], }) logger.warning("Reconcile: %s has unmanaged HL position %s (szi=%s, entry=%s)", sub.wallet_address, asset, hl_pos["szi"], hl_pos["entry_px"]) return summary # ─── Periodic top-level task ──────────────────────────────────────────────── async def reconcile_all_once() -> None: """One scan over every subscription. Wired into the APScheduler at startup.""" async with AsyncSessionLocal() as db: rows = await db.execute( select(Subscription).where( Subscription.active == True, # noqa: E712 Subscription.hl_api_key.is_not(None), ) ) subs = rows.scalars().all() if not subs: return # Snapshot the fields we read so we don't hold the session across HL calls. snapshots = list(subs) n_changed = 0 n_orphans = 0 n_errors = 0 for sub in snapshots: summary = await _reconcile_wallet(sub) n_changed += summary["marked_closed"] n_orphans += len(summary["orphan_hl"]) if summary["error"]: n_errors += 1 # Broadcast significant events so the UI shows a warning banner. if summary["marked_closed"] or summary["orphan_hl"]: try: from app.ws.manager import manager await manager.broadcast({ "type": "reconcile_drift", "wallet": summary["wallet"], "orphan_hl": summary["orphan_hl"], "marked_closed": summary["marked_closed"], "at": datetime.now(timezone.utc).isoformat() + "Z", }) except Exception as exc: logger.warning("reconcile WS broadcast failed: %s", exc) if n_changed or n_orphans or n_errors: logger.info("Reconcile cycle: %d subs scanned, %d trades marked closed, %d HL orphans, %d errors", len(snapshots), n_changed, n_orphans, n_errors)