k
d6c802ef26
Batch of the pre-launch audit campaign (BUG-01…14 plus three new features): Pricing / TP-SL protection - Add app/services/hl_price_feed.py: supplemental HL allMids poller for HL-native assets (HYPE, PURR) not listed on Binance. Pumps price_store + tp_sl_monitor.on_price_tick so bot trades on these assets keep full stop-loss / take-profit / trailing protection instead of max-hold only. - Wire feed into main.py lifespan (startup task + graceful shutdown cancel). Telegram - Add format_trump_mention + PATH B in _dispatch: crypto-relevant Trump posts with no directional signal (relevant=True, signal=hold) now alert the public channel only (no per-subscriber noise). - Rate limiter (slowapi) on the API; assorted bot/digest fixes. KOL on-chain - seed_kol_wallets.py: KOL_FEEDS coverage cross-check; reversibly deactivate orphaned wallets (handle not in KOL_FEEDS → can never produce divergence) so the scanner stops burning cycles on them. Tests / misc - Fix brittle test_macro_ahr999_uses_same_formula_as_scanner: mock now uses realistic ms timestamps so the in-progress-day drop fires, matching the fetcher's bar count (was 0.3179 vs 0.3178 off-by-one). - Refresh stale notify_signal comment in truth_social.py. Frontend reduce-action type fix lives in the sibling repo. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
271 lines
11 KiB
Python
271 lines
11 KiB
Python
"""
|
||
Hyperliquid <-> DB state reconciliation (P1.2).
|
||
|
||
Runs every RECONCILE_INTERVAL_SECONDS. For each subscription with a saved HL
|
||
API key (and NOT in paper mode), fetches actual open positions from HL and
|
||
compares them to BotTrade rows with closed_at IS NULL.
|
||
|
||
Two drift cases:
|
||
|
||
1. DB has open trade, HL has no matching position
|
||
→ User closed manually on HL UI, or HL liquidated, or our open_position
|
||
call failed silently after writing the row. Either way the row is stale.
|
||
Action: mark the DB row closed with pnl_usd=NULL, reason="closed_externally".
|
||
|
||
2. HL has open position, DB has no matching open trade
|
||
→ A trade exists on HL that we didn't open (legacy position, or stale
|
||
wallet shared across systems). We can't safely manage it — log a
|
||
warning + WS broadcast for the user to handle manually.
|
||
|
||
NOTE: this does NOT close stray HL positions. Auto-closing positions we didn't
|
||
open is exactly the kind of "helpful" behaviour that costs users money. We
|
||
report, the human decides.
|
||
|
||
Cost note:
|
||
HL's user_state endpoint is cheap. 1 call per active subscriber per
|
||
RECONCILE_INTERVAL. With 10 subscribers at 60s interval that's 600 calls/hr,
|
||
well inside HL's free tier.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Optional
|
||
|
||
from sqlalchemy import select, update
|
||
|
||
from app.config import settings
|
||
from app.database import AsyncSessionLocal
|
||
from app.models import BotTrade, Subscription
|
||
from app.services.crypto import decrypt_api_key
|
||
from app.services.hyperliquid import HyperliquidTrader
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
RECONCILE_INTERVAL_SECONDS = 60
|
||
|
||
# Max concurrent HL API calls per reconcile cycle. Each call can take up to
|
||
# ~30s on timeout; with 10 in flight the worst-case tail latency for N wallets
|
||
# is ceil(N/10) × 30s instead of N × 30s.
|
||
_RECONCILE_CONCURRENCY = 10
|
||
|
||
# How far back to look for the BUG-03 ghost-position check (DB closed but HL
|
||
# still open). 2 hours covers most close_and_finalize failure windows without
|
||
# scanning the entire trades history.
|
||
_GHOST_LOOKBACK_HOURS = 2
|
||
|
||
|
||
# ─── Single-wallet reconcile ────────────────────────────────────────────────
|
||
|
||
|
||
async def _reconcile_wallet(sub: Subscription) -> dict:
|
||
"""Compare HL state vs DB state for one subscription.
|
||
|
||
Returns a small summary dict for logging / WS broadcast. Never raises —
|
||
network / HL errors are logged and counted but don't halt the cycle.
|
||
"""
|
||
summary = {
|
||
"wallet": sub.wallet_address, "orphan_hl": [], "orphan_db": [],
|
||
"marked_closed": 0, "ghost_positions": [], "error": None,
|
||
}
|
||
|
||
# Paper-mode subs aren't on HL at all — nothing to reconcile.
|
||
if sub.paper_mode:
|
||
return summary
|
||
|
||
try:
|
||
api_key = decrypt_api_key(sub.hl_api_key)
|
||
except Exception as exc:
|
||
summary["error"] = f"decrypt_failed: {exc}"
|
||
return summary
|
||
|
||
try:
|
||
trader = HyperliquidTrader(
|
||
api_private_key=api_key,
|
||
account_address=sub.wallet_address,
|
||
leverage=sub.leverage,
|
||
mainnet=settings.hl_mainnet,
|
||
)
|
||
hl_positions = await trader.get_open_positions()
|
||
except Exception as exc:
|
||
summary["error"] = f"hl_query_failed: {exc}"
|
||
return summary
|
||
|
||
hl_assets_open = {p["coin"]: p for p in hl_positions}
|
||
|
||
async with AsyncSessionLocal() as db:
|
||
# All DB rows we believe are still open AND not released for this
|
||
# wallet. A released trade left the HL position open but handed
|
||
# control back to the user — reconciler must not touch it.
|
||
rows = await db.execute(
|
||
select(BotTrade).where(
|
||
BotTrade.wallet_address == sub.wallet_address,
|
||
BotTrade.closed_at.is_(None),
|
||
BotTrade.released_at.is_(None),
|
||
)
|
||
)
|
||
db_open_trades = rows.scalars().all()
|
||
db_assets_open = {t.asset: t for t in db_open_trades}
|
||
|
||
# Case 1: DB open trade, HL has nothing. Stale row → mark closed.
|
||
now_naive = datetime.now(timezone.utc).replace(tzinfo=None)
|
||
for asset, trade in db_assets_open.items():
|
||
# Paper trades never have HL positions; they're managed by the
|
||
# normal close path. Skip here.
|
||
if trade.hl_order_id == "paper":
|
||
continue
|
||
if asset not in hl_assets_open:
|
||
# Preserve any de-risk PnL ALREADY banked on this trade — the
|
||
# open remainder's PnL is unknown (closed externally) but the
|
||
# banked slice is real money on HL; setting pnl_usd=None would
|
||
# silently undercount realized PnL in analytics / KPIs.
|
||
banked = trade.realized_partial_pnl_usd or 0.0
|
||
preserved_pnl = round(banked, 2) if banked else None
|
||
# Atomic claim — same conditional UPDATE pattern as close_and_finalize.
|
||
claimed = await db.execute(
|
||
update(BotTrade)
|
||
.where(BotTrade.id == trade.id)
|
||
.where(BotTrade.closed_at.is_(None))
|
||
.values(closed_at=now_naive, pnl_usd=preserved_pnl,
|
||
remaining_fraction=0.0)
|
||
)
|
||
if claimed.rowcount > 0:
|
||
summary["marked_closed"] += 1
|
||
summary["orphan_db"].append({"asset": asset, "trade_id": trade.id})
|
||
# Stop the TP/SL watcher for this trade
|
||
from app.services.tp_sl_monitor import unregister
|
||
unregister(trade.id)
|
||
logger.warning("Reconcile: trade %d (%s %s) closed externally — marked.",
|
||
trade.id, trade.side, asset)
|
||
await db.commit()
|
||
|
||
# Case 2: HL has position we don't know about. Report only — don't auto-trade.
|
||
for asset, hl_pos in hl_assets_open.items():
|
||
if asset not in db_assets_open:
|
||
summary["orphan_hl"].append({
|
||
"asset": asset, "szi": hl_pos["szi"],
|
||
"entry": hl_pos["entry_px"], "uPnL": hl_pos["unrealized_pnl"],
|
||
})
|
||
logger.warning("Reconcile: %s has unmanaged HL position %s (szi=%s, entry=%s)",
|
||
sub.wallet_address, asset, hl_pos["szi"], hl_pos["entry_px"])
|
||
|
||
# Case 3 (BUG-03 mitigation): DB says closed recently but HL still shows
|
||
# the position open — the "ghost" state caused by close_and_finalize
|
||
# writing closed_at to DB while the HL market order fails. We can't
|
||
# auto-re-close (risk of double-execution) so we alert loudly and let
|
||
# the human decide. Look back _GHOST_LOOKBACK_HOURS only; older discrepancies
|
||
# are likely stale HL data or positions the user re-opened manually.
|
||
ghost_cutoff = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(
|
||
hours=_GHOST_LOOKBACK_HOURS
|
||
)
|
||
async with AsyncSessionLocal() as db:
|
||
ghost_rows = (await db.execute(
|
||
select(BotTrade).where(
|
||
BotTrade.wallet_address == sub.wallet_address,
|
||
BotTrade.closed_at.is_not(None),
|
||
BotTrade.closed_at >= ghost_cutoff,
|
||
)
|
||
)).scalars().all()
|
||
|
||
for trade in ghost_rows:
|
||
# Paper trades and adopted/released trades have no HL position to check.
|
||
if trade.hl_order_id == "paper" or trade.released_at is not None:
|
||
continue
|
||
if trade.asset in hl_assets_open:
|
||
ghost_entry = {
|
||
"asset": trade.asset,
|
||
"trade_id": trade.id,
|
||
"closed_at": trade.closed_at.isoformat() if trade.closed_at else None,
|
||
"hl_szi": hl_assets_open[trade.asset].get("szi"),
|
||
}
|
||
summary["ghost_positions"].append(ghost_entry)
|
||
logger.error(
|
||
"GHOST POSITION [BUG-03]: trade %d (%s %s) has closed_at=%s in DB "
|
||
"but HL still shows an open position (szi=%s). "
|
||
"Manual close required on HL UI.",
|
||
trade.id, trade.side, trade.asset, trade.closed_at,
|
||
hl_assets_open[trade.asset].get("szi"),
|
||
)
|
||
|
||
return summary
|
||
|
||
|
||
# ─── Periodic top-level task ────────────────────────────────────────────────
|
||
|
||
|
||
async def reconcile_all_once() -> None:
|
||
"""One scan over every subscription. Wired into the APScheduler at startup.
|
||
|
||
Runs all per-wallet checks concurrently (up to _RECONCILE_CONCURRENCY in
|
||
flight at once) so tail latency scales as ceil(N / concurrency) × per-call
|
||
timeout rather than N × per-call timeout.
|
||
"""
|
||
async with AsyncSessionLocal() as db:
|
||
rows = await db.execute(
|
||
select(Subscription).where(
|
||
Subscription.active == True, # noqa: E712
|
||
Subscription.hl_api_key.is_not(None),
|
||
)
|
||
)
|
||
subs = rows.scalars().all()
|
||
|
||
if not subs:
|
||
return
|
||
|
||
# Snapshot the fields we read so we don't hold the session across HL calls.
|
||
snapshots = list(subs)
|
||
|
||
sem = asyncio.Semaphore(_RECONCILE_CONCURRENCY)
|
||
|
||
async def _bounded(sub: Subscription) -> dict:
|
||
async with sem:
|
||
return await _reconcile_wallet(sub)
|
||
|
||
results = await asyncio.gather(
|
||
*[_bounded(sub) for sub in snapshots],
|
||
return_exceptions=True,
|
||
)
|
||
|
||
n_changed = 0
|
||
n_orphans = 0
|
||
n_ghosts = 0
|
||
n_errors = 0
|
||
for sub, result in zip(snapshots, results):
|
||
if isinstance(result, BaseException):
|
||
n_errors += 1
|
||
logger.error("Reconcile: unexpected exception for %s: %s",
|
||
sub.wallet_address, result)
|
||
continue
|
||
|
||
summary = result
|
||
n_changed += summary["marked_closed"]
|
||
n_orphans += len(summary["orphan_hl"])
|
||
n_ghosts += len(summary.get("ghost_positions", []))
|
||
if summary["error"]:
|
||
n_errors += 1
|
||
|
||
# Broadcast significant events so the UI shows a warning banner.
|
||
if summary["marked_closed"] or summary["orphan_hl"] or summary.get("ghost_positions"):
|
||
try:
|
||
from app.ws.manager import manager
|
||
await manager.broadcast({
|
||
"type": "reconcile_drift",
|
||
"wallet": summary["wallet"],
|
||
"orphan_hl": summary["orphan_hl"],
|
||
"marked_closed": summary["marked_closed"],
|
||
"ghost_positions": summary.get("ghost_positions", []),
|
||
"at": datetime.now(timezone.utc).isoformat() + "Z",
|
||
})
|
||
except Exception as exc:
|
||
logger.warning("reconcile WS broadcast failed: %s", exc)
|
||
|
||
if n_changed or n_orphans or n_ghosts or n_errors:
|
||
logger.info(
|
||
"Reconcile cycle: %d subs scanned, %d trades marked closed, "
|
||
"%d HL orphans, %d ghost positions, %d errors",
|
||
len(snapshots), n_changed, n_orphans, n_ghosts, n_errors,
|
||
)
|