""" Hyperliquid <-> DB state reconciliation (P1.2). Runs every RECONCILE_INTERVAL_SECONDS. For each subscription with a saved HL API key (and NOT in paper mode), fetches actual open positions from HL and compares them to BotTrade rows with closed_at IS NULL. Two drift cases: 1. DB has open trade, HL has no matching position → User closed manually on HL UI, or HL liquidated, or our open_position call failed silently after writing the row. Either way the row is stale. Action: mark the DB row closed with pnl_usd=NULL, reason="closed_externally". 2. HL has open position, DB has no matching open trade → A trade exists on HL that we didn't open (legacy position, or stale wallet shared across systems). We can't safely manage it — log a warning + WS broadcast for the user to handle manually. NOTE: this does NOT close stray HL positions. Auto-closing positions we didn't open is exactly the kind of "helpful" behaviour that costs users money. We report, the human decides. Cost note: HL's user_state endpoint is cheap. 1 call per active subscriber per RECONCILE_INTERVAL. With 10 subscribers at 60s interval that's 600 calls/hr, well inside HL's free tier. """ from __future__ import annotations import asyncio import logging from datetime import datetime, timedelta, timezone from typing import Optional from sqlalchemy import select, update from app.config import settings from app.database import AsyncSessionLocal from app.models import BotTrade, Subscription from app.services.crypto import decrypt_api_key from app.services.hyperliquid import HyperliquidTrader logger = logging.getLogger(__name__) RECONCILE_INTERVAL_SECONDS = 60 # Max concurrent HL API calls per reconcile cycle. Each call can take up to # ~30s on timeout; with 10 in flight the worst-case tail latency for N wallets # is ceil(N/10) × 30s instead of N × 30s. _RECONCILE_CONCURRENCY = 10 # How far back to look for the BUG-03 ghost-position check (DB closed but HL # still open). 2 hours covers most close_and_finalize failure windows without # scanning the entire trades history. _GHOST_LOOKBACK_HOURS = 2 # ─── Single-wallet reconcile ──────────────────────────────────────────────── async def _reconcile_wallet(sub: Subscription) -> dict: """Compare HL state vs DB state for one subscription. Returns a small summary dict for logging / WS broadcast. Never raises — network / HL errors are logged and counted but don't halt the cycle. """ summary = { "wallet": sub.wallet_address, "orphan_hl": [], "orphan_db": [], "marked_closed": 0, "ghost_positions": [], "error": None, } # Paper-mode subs aren't on HL at all — nothing to reconcile. if sub.paper_mode: return summary try: api_key = decrypt_api_key(sub.hl_api_key) except Exception as exc: summary["error"] = f"decrypt_failed: {exc}" return summary try: trader = HyperliquidTrader( api_private_key=api_key, account_address=sub.wallet_address, leverage=sub.leverage, mainnet=settings.hl_mainnet, ) hl_positions = await trader.get_open_positions() except Exception as exc: summary["error"] = f"hl_query_failed: {exc}" return summary hl_assets_open = {p["coin"]: p for p in hl_positions} async with AsyncSessionLocal() as db: # All DB rows we believe are still open AND not released for this # wallet. A released trade left the HL position open but handed # control back to the user — reconciler must not touch it. rows = await db.execute( select(BotTrade).where( BotTrade.wallet_address == sub.wallet_address, BotTrade.closed_at.is_(None), BotTrade.released_at.is_(None), ) ) db_open_trades = rows.scalars().all() db_assets_open = {t.asset: t for t in db_open_trades} # Case 1: DB open trade, HL has nothing. Stale row → mark closed. now_naive = datetime.now(timezone.utc).replace(tzinfo=None) for asset, trade in db_assets_open.items(): # Paper trades never have HL positions; they're managed by the # normal close path. Skip here. if trade.hl_order_id == "paper": continue if asset not in hl_assets_open: # Preserve any de-risk PnL ALREADY banked on this trade — the # open remainder's PnL is unknown (closed externally) but the # banked slice is real money on HL; setting pnl_usd=None would # silently undercount realized PnL in analytics / KPIs. banked = trade.realized_partial_pnl_usd or 0.0 preserved_pnl = round(banked, 2) if banked else None # Atomic claim — same conditional UPDATE pattern as close_and_finalize. claimed = await db.execute( update(BotTrade) .where(BotTrade.id == trade.id) .where(BotTrade.closed_at.is_(None)) .values(closed_at=now_naive, pnl_usd=preserved_pnl, remaining_fraction=0.0) ) if claimed.rowcount > 0: summary["marked_closed"] += 1 summary["orphan_db"].append({"asset": asset, "trade_id": trade.id}) # Stop the TP/SL watcher for this trade from app.services.tp_sl_monitor import unregister unregister(trade.id) logger.warning("Reconcile: trade %d (%s %s) closed externally — marked.", trade.id, trade.side, asset) await db.commit() # Case 2: HL has position we don't know about. Report only — don't auto-trade. for asset, hl_pos in hl_assets_open.items(): if asset not in db_assets_open: summary["orphan_hl"].append({ "asset": asset, "szi": hl_pos["szi"], "entry": hl_pos["entry_px"], "uPnL": hl_pos["unrealized_pnl"], }) logger.warning("Reconcile: %s has unmanaged HL position %s (szi=%s, entry=%s)", sub.wallet_address, asset, hl_pos["szi"], hl_pos["entry_px"]) # Case 3 (BUG-03 mitigation): DB says closed recently but HL still shows # the position open — the "ghost" state caused by close_and_finalize # writing closed_at to DB while the HL market order fails. We can't # auto-re-close (risk of double-execution) so we alert loudly and let # the human decide. Look back _GHOST_LOOKBACK_HOURS only; older discrepancies # are likely stale HL data or positions the user re-opened manually. ghost_cutoff = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta( hours=_GHOST_LOOKBACK_HOURS ) async with AsyncSessionLocal() as db: ghost_rows = (await db.execute( select(BotTrade).where( BotTrade.wallet_address == sub.wallet_address, BotTrade.closed_at.is_not(None), BotTrade.closed_at >= ghost_cutoff, ) )).scalars().all() for trade in ghost_rows: # Paper trades and adopted/released trades have no HL position to check. if trade.hl_order_id == "paper" or trade.released_at is not None: continue if trade.asset in hl_assets_open: ghost_entry = { "asset": trade.asset, "trade_id": trade.id, "closed_at": trade.closed_at.isoformat() if trade.closed_at else None, "hl_szi": hl_assets_open[trade.asset].get("szi"), } summary["ghost_positions"].append(ghost_entry) logger.error( "GHOST POSITION [BUG-03]: trade %d (%s %s) has closed_at=%s in DB " "but HL still shows an open position (szi=%s). " "Manual close required on HL UI.", trade.id, trade.side, trade.asset, trade.closed_at, hl_assets_open[trade.asset].get("szi"), ) return summary # ─── Periodic top-level task ──────────────────────────────────────────────── async def reconcile_all_once() -> None: """One scan over every subscription. Wired into the APScheduler at startup. Runs all per-wallet checks concurrently (up to _RECONCILE_CONCURRENCY in flight at once) so tail latency scales as ceil(N / concurrency) × per-call timeout rather than N × per-call timeout. """ async with AsyncSessionLocal() as db: rows = await db.execute( select(Subscription).where( Subscription.active == True, # noqa: E712 Subscription.hl_api_key.is_not(None), ) ) subs = rows.scalars().all() if not subs: return # Snapshot the fields we read so we don't hold the session across HL calls. snapshots = list(subs) sem = asyncio.Semaphore(_RECONCILE_CONCURRENCY) async def _bounded(sub: Subscription) -> dict: async with sem: return await _reconcile_wallet(sub) results = await asyncio.gather( *[_bounded(sub) for sub in snapshots], return_exceptions=True, ) n_changed = 0 n_orphans = 0 n_ghosts = 0 n_errors = 0 for sub, result in zip(snapshots, results): if isinstance(result, BaseException): n_errors += 1 logger.error("Reconcile: unexpected exception for %s: %s", sub.wallet_address, result) continue summary = result n_changed += summary["marked_closed"] n_orphans += len(summary["orphan_hl"]) n_ghosts += len(summary.get("ghost_positions", [])) if summary["error"]: n_errors += 1 # Broadcast significant events so the UI shows a warning banner. if summary["marked_closed"] or summary["orphan_hl"] or summary.get("ghost_positions"): try: from app.ws.manager import manager await manager.broadcast({ "type": "reconcile_drift", "wallet": summary["wallet"], "orphan_hl": summary["orphan_hl"], "marked_closed": summary["marked_closed"], "ghost_positions": summary.get("ghost_positions", []), "at": datetime.now(timezone.utc).isoformat() + "Z", }) except Exception as exc: logger.warning("reconcile WS broadcast failed: %s", exc) if n_changed or n_orphans or n_ghosts or n_errors: logger.info( "Reconcile cycle: %d subs scanned, %d trades marked closed, " "%d HL orphans, %d ghost positions, %d errors", len(snapshots), n_changed, n_orphans, n_ghosts, n_errors, )