fix: pre-launch hardening — HYPE price feed, KOL wallet cleanup, Telegram Trump alert, rate limiting, brittle test

Batch of the pre-launch audit campaign (BUG-01…14 plus three new features): Pricing / TP-SL protection - Add app/services/hl_price_feed.py: supplemental HL allMids poller for HL-native assets (HYPE, PURR) not listed on Binance. Pumps price_store + tp_sl_monitor.on_price_tick so bot trades on these assets keep full stop-loss / take-profit / trailing protection instead of max-hold only. - Wire feed into main.py lifespan (startup task + graceful shutdown cancel). Telegram - Add format_trump_mention + PATH B in _dispatch: crypto-relevant Trump posts with no directional signal (relevant=True, signal=hold) now alert the public channel only (no per-subscriber noise). - Rate limiter (slowapi) on the API; assorted bot/digest fixes. KOL on-chain - seed_kol_wallets.py: KOL_FEEDS coverage cross-check; reversibly deactivate orphaned wallets (handle not in KOL_FEEDS → can never produce divergence) so the scanner stops burning cycles on them. Tests / misc - Fix brittle test_macro_ahr999_uses_same_formula_as_scanner: mock now uses realistic ms timestamps so the in-progress-day drop fires, matching the fetcher's bar count (was 0.3179 vs 0.3178 off-by-one). - Refresh stale notify_signal comment in truth_social.py. Frontend reduce-action type fix lives in the sibling repo. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-29 11:57:19 +08:00
parent 6471e44aac
commit d6c802ef26
40 changed files with 1833 additions and 209 deletions
@@ -1,8 +1,12 @@
 """
-把 AI 信号写回数据库。
-跳过纯RT/URL帖子和已有signal的帖子。
-并发执行提速，自动限速避免API超限。
-用法: python scripts/backfill_signals.py [--limit 500] [--overwrite]
+Backfill AI analysis onto historical Truth posts.
+
+Safety rules:
+  * Only re-analyze `source='truth'` rows. Technical/scanner posts already
+    carry their own signal payloads and must not be sent through the Trump
+    text analyzer.
+  * Persist the full analysis payload so history doesn't become a mixed schema
+    of old partial rows and new v5 rows.
 """
 import asyncio
 import argparse
@@ -38,8 +42,11 @@ async def process_one(post: Post, semaphore: asyncio.Semaphore, overwrite: bool)
            p.relevant          = analysis["relevant"]
            p.prefilter_reason  = analysis.get("prefilter_reason")
            p.analysis_version  = analysis.get("analysis_version")
-            if analysis["relevant"] and analysis["asset"] and not p.price_impact_asset:
-                p.price_impact_asset = analysis["asset"]
+            p.price_impact_asset = analysis["asset"] if analysis["relevant"] else None
+            p.target_asset = analysis.get("target_asset")
+            p.category = analysis.get("category")
+            p.expected_move_pct = analysis.get("expected_move_pct")
+            p.invalidation_price = analysis.get("invalidation_price")
            await db.commit()

    sig = analysis["signal"]
@@ -49,7 +56,10 @@ async def process_one(post: Post, semaphore: asyncio.Semaphore, overwrite: bool)
 async def main(limit: int, overwrite: bool):
    async with AsyncSessionLocal() as db:
        result = await db.execute(
-            select(Post).order_by(Post.published_at.desc()).limit(limit)
+            select(Post)
+            .where(Post.source == "truth")
+            .order_by(Post.published_at.desc())
+            .limit(limit)
        )
        posts = result.scalars().all()

@@ -13,7 +13,7 @@ What it does:
       * KOL Substack / podcast / blog polls
       * KOL on-chain snapshots (HL perps + Etherscan ERC-20 balances)
       * KOL divergence detector
-       * BTC bottom-reversal + funding-reversal scanners
+       * Optional: BTC bottom-reversal + funding-reversal scanners

 What it INTENTIONALLY DOES NOT touch:
  * posts where source IN ('truth', 'btc_bottom_reversal', 'funding_reversal',
@@ -33,6 +33,10 @@ Usage:
    DATABASE_URL='sqlite+aiosqlite:///./trumpsignal.db' \\
        venv/bin/python scripts/launch_seed.py --dry-run

+    # Seed only (pure pre-launch warmup, no truncation / deletion):
+    DATABASE_URL='sqlite+aiosqlite:///./trumpsignal.db' \\
+        venv/bin/python scripts/launch_seed.py --seed-only
+
    # Execute:
    DATABASE_URL='sqlite+aiosqlite:///./trumpsignal.db' \\
        venv/bin/python scripts/launch_seed.py --yes
@@ -198,7 +202,29 @@ async def wipe_phase(dry_run: bool) -> None:
        print(green("  ✓ wipe committed"))


-async def seed_real_data() -> None:
+async def abort_if_live_user_state() -> bool:
+    """Refuse destructive launch seeding if the DB already looks user-live."""
+    async with AsyncSessionLocal() as db:
+        active_subs = (await db.execute(
+            select(func.count(Subscription.id)).where(Subscription.active == True)
+        )).scalar() or 0
+        tg_bindings = (await db.execute(
+            select(func.count(TelegramBinding.id))
+        )).scalar() or 0
+    if active_subs or tg_bindings:
+        print(red(
+            "Refusing to run launch_seed on a DB that already has live user state "
+            f"(active_subscriptions={active_subs}, telegram_bindings={tg_bindings})."
+        ))
+        print(yellow(
+            "Use --dry-run to inspect only. If you truly intend to clean this DB, "
+            "do it manually with a one-off migration/backup plan."
+        ))
+        return True
+    return False
+
+
+async def seed_real_data(*, exercise_scanners: bool = False) -> None:
    """Re-fetch every upstream source. All are idempotent on (source, external_id)."""
    print(bold("\n── SEED (re-running upstream fetches) ──"))

@@ -237,15 +263,21 @@ async def seed_real_data() -> None:
    except Exception as e:
        print(red(f"        ✗ FAILED: {type(e).__name__}: {e}"))

-    print("\n  [5/5] BTC bottom + funding reversal scanners (exercise the path)...")
-    from app.services.scanners.btc_bottom_reversal import scan_once as btc_scan
-    from app.services.scanners.funding_reversal import scan_once as funding_scan
-    for name, fn in [("btc_bottom", btc_scan), ("funding_reversal", funding_scan)]:
-        try:
-            await fn()
-            print(green(f"        ✓ {name} scan completed (fire conditional on market state)"))
-        except Exception as e:
-            print(red(f"        ✗ {name} FAILED: {type(e).__name__}: {e}"))
+    if exercise_scanners:
+        print("\n  [5/5] BTC bottom + funding reversal scanners (exercise the path)...")
+        from app.services.scanners.btc_bottom_reversal import scan_once as btc_scan
+        from app.services.scanners.funding_reversal import scan_once as funding_scan
+        for name, fn in [("btc_bottom", btc_scan), ("funding_reversal", funding_scan)]:
+            try:
+                await fn()
+                print(green(f"        ✓ {name} scan completed (fire conditional on market state)"))
+            except Exception as e:
+                print(red(f"        ✗ {name} FAILED: {type(e).__name__}: {e}"))
+    else:
+        print(yellow(
+            "\n  [5/5] Scanner exercise skipped by default. "
+            "Use --exercise-scanners only if you explicitly want signal emission side-effects."
+        ))


 async def main() -> int:
@@ -253,16 +285,46 @@ async def main() -> int:
                                formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--dry-run", action="store_true",
                   help="show what would be deleted, no DB writes")
+    p.add_argument("--seed-only", action="store_true",
+                   help="run upstream fetch/warmup only; skip all wipe/truncation steps")
    p.add_argument("--yes", action="store_true",
                   help="actually perform the wipe (required without --dry-run)")
    p.add_argument("--no-seed", action="store_true",
                   help="skip the upstream re-fetch step")
+    p.add_argument("--exercise-scanners", action="store_true",
+                   help="also run standalone scanners during seed (may emit signals/alerts)")
    args = p.parse_args()

+    if args.seed_only and args.dry_run:
+        print(red("Choose either --dry-run or --seed-only, not both."))
+        return 2
+
+    if args.seed_only and args.no_seed:
+        print(red("--seed-only conflicts with --no-seed."))
+        return 2
+
+    if args.seed_only:
+        before = await report_counts("BEFORE")
+        await seed_real_data(exercise_scanners=args.exercise_scanners)
+        after = await report_counts("AFTER")
+
+        print(bold("\n── DELTA ──"))
+        for k in sorted(before):
+            if k.startswith("_"): continue
+            d = after[k] - before[k]
+            arrow = green(f"+{d}") if d > 0 else (red(str(d)) if d < 0 else "  ·")
+            print(f"  {k:25s} {before[k]:6d} → {after[k]:6d}  ({arrow})")
+
+        print(bold(green("\n✓ seed-only warmup complete. Next: start backend and run scripts/launch_smoke.py.")))
+        return 0
+
    if not args.dry_run and not args.yes:
        print(red("Refusing to run without --yes (or use --dry-run to preview)."))
        return 2

+    if not args.dry_run and await abort_if_live_user_state():
+        return 2
+
    before = await report_counts("BEFORE")
    await wipe_phase(dry_run=args.dry_run)

@@ -271,7 +333,7 @@ async def main() -> int:
        return 0

    if not args.no_seed:
-        await seed_real_data()
+        await seed_real_data(exercise_scanners=args.exercise_scanners)

    after = await report_counts("AFTER")

@@ -73,6 +73,16 @@ SEED_WALLETS: list[dict] = [
        "label": "Arthur Hayes (secondary)",
        "source_url": "https://etherscan.io/address/0x534a0076fb7c2b1f83fa21497429ad7ad3bd7587",
    },
+    # ⚠️ ORPHANED: the two entries below have VERIFIED on-chain attribution
+    # but their `handle` does NOT match any handle in KOL_FEEDS, so the
+    # divergence scanner has no post-side data to join against — they produce
+    # ZERO divergence detections today. They are kept (a) because the wallet
+    # attribution is sound and (b) so that if/when X/Twitter ingestion or a
+    # matching RSS feed is added under these handles, the wallets light up
+    # automatically. The cross-check in main() prints a warning for these.
+    #   • andrewkang — publishes on X/@Rewkang only (no Substack) → needs
+    #                   X ingestion before post-side data exists.
+    #   • murad      — not currently in KOL_FEEDS at all.
    {
        "handle": "andrewkang",
        "chain": "ethereum",
@@ -152,6 +162,57 @@ async def main() -> int:

    print()
    print(f"Inserted {inserted} wallets, skipped {skipped} existing.")
+
+    # ── Coverage cross-check ─────────────────────────────────────────────
+    # A wallet is only useful for divergence detection if its handle has a
+    # matching post-side feed in KOL_FEEDS. Flag any orphans loudly, and
+    # report which feeds still have no wallet at all (the real coverage gap).
+    try:
+        from app.services.kol_substack import KOL_FEEDS
+        feed_handles = {f["handle"] for f in KOL_FEEDS}
+    except Exception as exc:  # pragma: no cover - diagnostic only
+        print(f"\n(could not import KOL_FEEDS for cross-check: {exc})")
+        return 0
+
+    wallet_handles = {e["handle"] for e in SEED_WALLETS}
+    orphaned = sorted(wallet_handles - feed_handles)
+    feeds_without_wallet = sorted(feed_handles - wallet_handles)
+
+    print()
+    print(f"Coverage: {len(wallet_handles - set(orphaned))}/{len(feed_handles)} "
+          f"KOL_FEEDS handles have ≥1 wallet.")
+
+    # ── Orphan reconciliation ────────────────────────────────────────────
+    # An orphaned wallet (handle ∉ KOL_FEEDS) can NEVER produce a divergence
+    # because there's no post-side data to join against — but kol_onchain
+    # still burns a scan cycle (HL clearinghouseState + Etherscan) on it every
+    # run. Park such rows as active=False so the scanner skips them. This is
+    # REVERSIBLE: the row + its verified attribution stay in the DB, and the
+    # moment a matching feed (e.g. X/Twitter ingestion) is added, flip it back.
+    if orphaned:
+        async with AsyncSessionLocal() as session:
+            res = await session.execute(
+                select(KolWallet).where(
+                    KolWallet.handle.in_(orphaned),
+                    KolWallet.active.is_(True),
+                )
+            )
+            rows = res.scalars().all()
+            for row in rows:
+                row.active = False
+            await session.commit()
+            deactivated = len(rows)
+        print()
+        print(f"⚠️  ORPHANED wallets (handle not in KOL_FEEDS → ZERO divergence) "
+              f"— deactivated {deactivated} so the scanner skips them:")
+        for h in orphaned:
+            print(f"      {h}  — re-activate once a feed under this handle exists.")
+    if feeds_without_wallet:
+        print()
+        print(f"📭 {len(feeds_without_wallet)} feeds have NO wallet (no on-chain "
+              f"divergence possible):")
+        print("      " + ", ".join(feeds_without_wallet))
+
    print()
    print("Next step: edit SEED_WALLETS above and re-run. Each new wallet")
    print("MUST cite a public attestation in source_url — see the docstring.")