fix: pre-launch hardening — HYPE price feed, KOL wallet cleanup, Telegram Trump alert, rate limiting, brittle test
Batch of the pre-launch audit campaign (BUG-01…14 plus three new features): Pricing / TP-SL protection - Add app/services/hl_price_feed.py: supplemental HL allMids poller for HL-native assets (HYPE, PURR) not listed on Binance. Pumps price_store + tp_sl_monitor.on_price_tick so bot trades on these assets keep full stop-loss / take-profit / trailing protection instead of max-hold only. - Wire feed into main.py lifespan (startup task + graceful shutdown cancel). Telegram - Add format_trump_mention + PATH B in _dispatch: crypto-relevant Trump posts with no directional signal (relevant=True, signal=hold) now alert the public channel only (no per-subscriber noise). - Rate limiter (slowapi) on the API; assorted bot/digest fixes. KOL on-chain - seed_kol_wallets.py: KOL_FEEDS coverage cross-check; reversibly deactivate orphaned wallets (handle not in KOL_FEEDS → can never produce divergence) so the scanner stops burning cycles on them. Tests / misc - Fix brittle test_macro_ahr999_uses_same_formula_as_scanner: mock now uses realistic ms timestamps so the in-progress-day drop fires, matching the fetcher's bar count (was 0.3179 vs 0.3178 off-by-one). - Refresh stale notify_signal comment in truth_social.py. Frontend reduce-action type fix lives in the sibling repo. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
k
+90
-14
@@ -31,7 +31,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select, update
|
||||
@@ -47,6 +47,16 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
RECONCILE_INTERVAL_SECONDS = 60
|
||||
|
||||
# Max concurrent HL API calls per reconcile cycle. Each call can take up to
|
||||
# ~30s on timeout; with 10 in flight the worst-case tail latency for N wallets
|
||||
# is ceil(N/10) × 30s instead of N × 30s.
|
||||
_RECONCILE_CONCURRENCY = 10
|
||||
|
||||
# How far back to look for the BUG-03 ghost-position check (DB closed but HL
|
||||
# still open). 2 hours covers most close_and_finalize failure windows without
|
||||
# scanning the entire trades history.
|
||||
_GHOST_LOOKBACK_HOURS = 2
|
||||
|
||||
|
||||
# ─── Single-wallet reconcile ────────────────────────────────────────────────
|
||||
|
||||
@@ -59,7 +69,7 @@ async def _reconcile_wallet(sub: Subscription) -> dict:
|
||||
"""
|
||||
summary = {
|
||||
"wallet": sub.wallet_address, "orphan_hl": [], "orphan_db": [],
|
||||
"marked_closed": 0, "error": None,
|
||||
"marked_closed": 0, "ghost_positions": [], "error": None,
|
||||
}
|
||||
|
||||
# Paper-mode subs aren't on HL at all — nothing to reconcile.
|
||||
@@ -142,6 +152,44 @@ async def _reconcile_wallet(sub: Subscription) -> dict:
|
||||
logger.warning("Reconcile: %s has unmanaged HL position %s (szi=%s, entry=%s)",
|
||||
sub.wallet_address, asset, hl_pos["szi"], hl_pos["entry_px"])
|
||||
|
||||
# Case 3 (BUG-03 mitigation): DB says closed recently but HL still shows
|
||||
# the position open — the "ghost" state caused by close_and_finalize
|
||||
# writing closed_at to DB while the HL market order fails. We can't
|
||||
# auto-re-close (risk of double-execution) so we alert loudly and let
|
||||
# the human decide. Look back _GHOST_LOOKBACK_HOURS only; older discrepancies
|
||||
# are likely stale HL data or positions the user re-opened manually.
|
||||
ghost_cutoff = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(
|
||||
hours=_GHOST_LOOKBACK_HOURS
|
||||
)
|
||||
async with AsyncSessionLocal() as db:
|
||||
ghost_rows = (await db.execute(
|
||||
select(BotTrade).where(
|
||||
BotTrade.wallet_address == sub.wallet_address,
|
||||
BotTrade.closed_at.is_not(None),
|
||||
BotTrade.closed_at >= ghost_cutoff,
|
||||
)
|
||||
)).scalars().all()
|
||||
|
||||
for trade in ghost_rows:
|
||||
# Paper trades and adopted/released trades have no HL position to check.
|
||||
if trade.hl_order_id == "paper" or trade.released_at is not None:
|
||||
continue
|
||||
if trade.asset in hl_assets_open:
|
||||
ghost_entry = {
|
||||
"asset": trade.asset,
|
||||
"trade_id": trade.id,
|
||||
"closed_at": trade.closed_at.isoformat() if trade.closed_at else None,
|
||||
"hl_szi": hl_assets_open[trade.asset].get("szi"),
|
||||
}
|
||||
summary["ghost_positions"].append(ghost_entry)
|
||||
logger.error(
|
||||
"GHOST POSITION [BUG-03]: trade %d (%s %s) has closed_at=%s in DB "
|
||||
"but HL still shows an open position (szi=%s). "
|
||||
"Manual close required on HL UI.",
|
||||
trade.id, trade.side, trade.asset, trade.closed_at,
|
||||
hl_assets_open[trade.asset].get("szi"),
|
||||
)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
@@ -149,7 +197,12 @@ async def _reconcile_wallet(sub: Subscription) -> dict:
|
||||
|
||||
|
||||
async def reconcile_all_once() -> None:
|
||||
"""One scan over every subscription. Wired into the APScheduler at startup."""
|
||||
"""One scan over every subscription. Wired into the APScheduler at startup.
|
||||
|
||||
Runs all per-wallet checks concurrently (up to _RECONCILE_CONCURRENCY in
|
||||
flight at once) so tail latency scales as ceil(N / concurrency) × per-call
|
||||
timeout rather than N × per-call timeout.
|
||||
"""
|
||||
async with AsyncSessionLocal() as db:
|
||||
rows = await db.execute(
|
||||
select(Subscription).where(
|
||||
@@ -165,30 +218,53 @@ async def reconcile_all_once() -> None:
|
||||
# Snapshot the fields we read so we don't hold the session across HL calls.
|
||||
snapshots = list(subs)
|
||||
|
||||
sem = asyncio.Semaphore(_RECONCILE_CONCURRENCY)
|
||||
|
||||
async def _bounded(sub: Subscription) -> dict:
|
||||
async with sem:
|
||||
return await _reconcile_wallet(sub)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*[_bounded(sub) for sub in snapshots],
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
n_changed = 0
|
||||
n_orphans = 0
|
||||
n_ghosts = 0
|
||||
n_errors = 0
|
||||
for sub in snapshots:
|
||||
summary = await _reconcile_wallet(sub)
|
||||
for sub, result in zip(snapshots, results):
|
||||
if isinstance(result, BaseException):
|
||||
n_errors += 1
|
||||
logger.error("Reconcile: unexpected exception for %s: %s",
|
||||
sub.wallet_address, result)
|
||||
continue
|
||||
|
||||
summary = result
|
||||
n_changed += summary["marked_closed"]
|
||||
n_orphans += len(summary["orphan_hl"])
|
||||
n_ghosts += len(summary.get("ghost_positions", []))
|
||||
if summary["error"]:
|
||||
n_errors += 1
|
||||
|
||||
# Broadcast significant events so the UI shows a warning banner.
|
||||
if summary["marked_closed"] or summary["orphan_hl"]:
|
||||
if summary["marked_closed"] or summary["orphan_hl"] or summary.get("ghost_positions"):
|
||||
try:
|
||||
from app.ws.manager import manager
|
||||
await manager.broadcast({
|
||||
"type": "reconcile_drift",
|
||||
"wallet": summary["wallet"],
|
||||
"orphan_hl": summary["orphan_hl"],
|
||||
"marked_closed": summary["marked_closed"],
|
||||
"at": datetime.now(timezone.utc).isoformat() + "Z",
|
||||
"type": "reconcile_drift",
|
||||
"wallet": summary["wallet"],
|
||||
"orphan_hl": summary["orphan_hl"],
|
||||
"marked_closed": summary["marked_closed"],
|
||||
"ghost_positions": summary.get("ghost_positions", []),
|
||||
"at": datetime.now(timezone.utc).isoformat() + "Z",
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.warning("reconcile WS broadcast failed: %s", exc)
|
||||
|
||||
if n_changed or n_orphans or n_errors:
|
||||
logger.info("Reconcile cycle: %d subs scanned, %d trades marked closed, %d HL orphans, %d errors",
|
||||
len(snapshots), n_changed, n_orphans, n_errors)
|
||||
if n_changed or n_orphans or n_ghosts or n_errors:
|
||||
logger.info(
|
||||
"Reconcile cycle: %d subs scanned, %d trades marked closed, "
|
||||
"%d HL orphans, %d ghost positions, %d errors",
|
||||
len(snapshots), n_changed, n_orphans, n_ghosts, n_errors,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user