From 78fb63be8e2d11122ec4821b8135606a7689ecf8 Mon Sep 17 00:00:00 2001 From: k Date: Sun, 14 Jun 2026 21:43:43 +0800 Subject: [PATCH] improve signed reads, crypto hardening, and scraper transport --- AGENTS.md | 609 +++++++++++++++++++++++++++++ CLAUDE.md | 90 ++++- app/api/performance.py | 10 +- app/api/positions.py | 25 +- app/api/prices.py | 7 +- app/api/telegram.py | 12 +- app/api/trades.py | 10 +- app/api/user.py | 10 +- app/main.py | 19 +- app/scrapers/trumpstruth.py | 81 ++-- app/scrapers/truth_social.py | 129 ++++-- app/services/adoption.py | 10 + app/services/binance.py | 27 +- app/services/crypto.py | 135 +++++-- app/services/hl_price_feed.py | 9 +- app/services/http_client.py | 37 ++ app/services/kol_analysis.py | 29 ++ app/services/kol_substack.py | 3 + app/services/signed_request.py | 55 +++ app/services/telegram.py | 13 +- app/services/telegram_bot.py | 11 +- app/services/x_poster.py | 5 +- scripts/reencrypt_keys.py | 90 +++++ tests/test_adoption.py | 2 +- tests/test_crypto.py | 58 +++ tests/test_kol_tier.py | 38 ++ tests/test_production_readiness.py | 4 +- 27 files changed, 1326 insertions(+), 202 deletions(-) create mode 100644 AGENTS.md create mode 100644 app/services/http_client.py create mode 100644 scripts/reencrypt_keys.py create mode 100644 tests/test_crypto.py create mode 100644 tests/test_kol_tier.py diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..f3b2504 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,609 @@ +# Trump Alpha โ€” Backend + +> AI-powered crypto signal aggregator. Surfaces four uncorrelated signal +> streams (Trump Truth Social, Macro Vibes, KOL talks-vs-trades, funding +> reversal) and runs an optional execution layer on Hyperliquid perps. +> Real money โ€” handle every change to the trading layer like surgery. + +This file is the **first thing an AI agent should read** when entering this +repo. It encodes the invariants that aren't visible from any single file. + +--- + +## ๐Ÿ›‘ Read this BEFORE touching anything trading-related + +This backend manages real Hyperliquid leveraged positions for real users. +Bugs cost users money. **Five non-negotiable rules:** + +1. **Two systems, one execution layer.** System 1 (Trump scalp) auto-opens. + System 2 (Macro Vibes / reversal) is **MANAGE-ONLY since v2.0** โ€” the + user opens manually on Hyperliquid, then `/adopt` hands it to the bot. + `process_post()` early-returns for sys2 sources. Re-enabling sys2 + auto-open would silently reintroduce all the leverage/budget/concurrency + race conditions we excised. Don't do it without an ADR. + +2. **`released_at` is the "user took back control" marker.** A trade with + `released_at IS NOT NULL` is **OUT OF BOUNDS for the bot**: + - `recovery.rehydrate_open_trades` skips it + - `reconciler` skips it + - `close_and_finalize`'s atomic claim requires `released_at IS NULL` + unless `force=True` (only manual_close passes that) + - `partial_derisk` and `pyramid_add` early-return idempotent-success + - `/positions/open`, `/positions/today`, telegram_digest all filter it + If you add ANY new code path that touches BotTrade rows, ask yourself + "does this respect released_at?" Almost always yes. + +3. **Effective exit params are FROZEN on the BotTrade row at open time.** + See the `eff_*` columns on `BotTrade`. Recovery rebuilds the watchdog + from these, NOT from the live Subscription. Without this, restarting + the backend silently rewrites every open System-2 reversal's stop loss + to the user's Trump scalp setting (1.5%). NEVER read live + `Subscription.stop_loss_pct` etc. in the close path. + +4. **HL leverage is what HL says, not what the user requested.** Hyperliquid + silently clips the requested leverage to the asset's max (memes capped + at 3ร—). `hyperliquid.open_position()` returns `effective_leverage` โ€” + bot_engine and adoption must use THAT value to compute + `sys2_protective_stop_pct(lev)` and the derisk ladder. Using the + requested value puts the stop OUTSIDE the real liquidation line. + +5. **Per-wallet asyncio locks wrap "check + open" critical sections.** + See `_wallet_lock` in `bot_engine` and `_wallet_adopt_lock` in + `adoption`. Without them, two concurrent signals can both pass the + daily-budget / concurrency / already-open check before either commits. + +--- + +## What this product does (90 seconds) + +``` +Four signal sources โ†’ one bot โ†’ optional Hyperliquid execution + +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ 1. Trump Truth Social โ”‚โ”€โ”€ auto-classify (DeepSeek) โ†’ "buy"/"short"/"noise" +โ”‚ (every post, <3s) โ”‚ if actionable: Trump scalp auto-open (System 1) +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ Tight 1.5% SL, 12h cooldown, โ‰ฅ30min min-hold + Optional: post prediction tweet via x_poster.py + +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ 2. Macro Vibes โ”‚โ”€โ”€ 8 daily macro indicators (AHR999, F&G, etc.) +โ”‚ (BTC bottom + funding)โ”‚ + 2-of-3 bottom-reversal trigger +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ Telegram alert with /adopt CTA โ€” NO auto-open. + User opens on HL โ†’ /adopt โ†’ bot manages with + 5-rung stop ladder, de-risk, pyramid, peak-trail. + +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ 3. KOL talks-vs-trades โ”‚โ”€โ”€ Substack/podcast/X ingest + ETH on-chain diff +โ”‚ (29 feeds, daily) โ”‚ Divergence (publicly bullish, secretly selling) +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ is the platform's highest-conviction signal. + Telegram alert only โ€” never auto-trades. + x_analysis.py adds real-time X post scoring. + +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ 4. Funding extreme โ”‚โ”€โ”€ Hourly BTC perp funding scan +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ Alert only (manage-only via /adopt like Macro) + +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Telegram daily digest โ”‚โ”€โ”€ Once-a-day 3-section brief (Macro/KOL/Trump) +โ”‚ (per-user hour, opt-out)โ”‚ to every subscriber. Cron at minute=0 each hour. +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +Free tier = read everything + Telegram alerts. Pro tier (Hyperliquid wallet +linked) = Trump auto-trade + /adopt manage-only flow for sys2. + +--- + +## Stack + +- **Python 3.9+** / FastAPI / async SQLAlchemy 2.x / APScheduler +- **DB**: SQLite dev, **Postgres prod**. All schema lives in + `alembic/versions/NNN_*.py`, ordered. Currently at head **026**. +- **AI**: DeepSeek via OpenAI-compatible API (`AI_BASE_URL`, `AI_MODEL`). + - Live scoring uses `AI_LIVE_MODEL` (~2s, latency-critical) + - Batch / reanalysis uses `AI_MODEL` (quality, ~10s) +- **Trading**: Hyperliquid SDK; API-wallet keys are envelope-encrypted with + `ENCRYPTION_KEY` (KEK), per-user DEK derivation via `crypto.py`. +- **Prices**: Two feeds: + - `binance.py` WebSocket โ†’ 30 mainstream perps (BTC, ETH, SOL, TRUMP, BNB, + DOGE, LINK, AAVE + AVAX/ARB/OP/SUI/APT/INJ/ATOM/XRP/LTC/ADA/MATIC/SHIB/ + PEPE/WIF/BONK/TAO/JUP/RENDER/FET/TIA/SEI/PENDLE). The WS URL is built FROM + `ASSET_MAP` so the two never drift. Any asset NOT in ASSET_MAP loses + TP/SL/trailing protection (only max-hold remains) โ€” see the ASSET_MAP + docstring. `tp_sl_monitor.register_trade` logs an ERROR if a trade opens on + an uncovered asset. + - `hl_price_feed.py` polls HL `allMids` every 2s โ†’ HYPE, PURR (HL-native assets not on Binance) + Both pump `price_store` + `tp_sl_monitor` on every tick. +- **Telegram**: long-poll mode (single instance), HTML messages, inline + keyboards. `telegram.py` send/edit/answer + `telegram_bot.py` commands. + Public channel broadcast (`TELEGRAM_PUBLIC_CHANNEL_ID` env) sends a + sanitised `format_public_post` version (no execution details, tier label + instead of raw confidence) after every per-user fan-out in `_dispatch`. +- **X (Twitter)**: `x_poster.py` โ€” optional viral prediction tweets after each + actionable Trump signal. Gated by `x_enabled=False` (off by default). Full + no-op if creds missing. OAuth 1.0a hand-signed with stdlib hmac โ€” no extra deps. + +--- + +## Module map (where things live) + +``` +app/ +โ”œโ”€โ”€ api/ HTTP routes +โ”‚ โ”œโ”€โ”€ signals.py POST /api/signals/ingest โ† scanners write here +โ”‚ โ”œโ”€โ”€ positions.py /positions/open|today|close|grow|adopt|release|hl +โ”‚ โ”œโ”€โ”€ user.py /subscribe|settings|manual-window|auto-trade +โ”‚ โ”œโ”€โ”€ telegram.py /telegram/{preferences,bind,unbind,test} +โ”‚ โ”œโ”€โ”€ macro.py /macro/{snapshot,history} +โ”‚ โ”œโ”€โ”€ kol.py /kol/{posts,digest,wallets,divergence} +โ”‚ โ”œโ”€โ”€ performance.py /performance โ† wallet-scoped real-money stats (30d) +โ”‚ โ”œโ”€โ”€ funding_reversal.py /funding/snapshot โ† live funding state + 7d history +โ”‚ โ”œโ”€โ”€ funding_signal.py /signal/{status,toggle,history} โ† breakout monitor +โ”‚ โ””โ”€โ”€ dev.py Dev-only routes (only mounted in development env) +โ”œโ”€โ”€ services/ +โ”‚ โ”œโ”€โ”€ bot_engine.py โ˜… TRADING CORE โ€” process_post, _execute_for_subscriber, +โ”‚ โ”‚ _broadcast_trade_alert (WS failure notifications), +โ”‚ โ”‚ close_and_finalize, partial_derisk, pyramid_add +โ”‚ โ”œโ”€โ”€ adoption.py โ˜… /adopt + /release flow (sys2 manage-only) +โ”‚ โ”œโ”€โ”€ tp_sl_monitor.py Per-price-tick close evaluator. on_price_tick is +โ”‚ โ”‚ called from binance.py + hl_price_feed.py once/sec +โ”‚ โ”œโ”€โ”€ hyperliquid.py HL trader (open/close/reduce/leverage) +โ”‚ โ”œโ”€โ”€ recovery.py Startup rehydration of open BotTrades into watchdog +โ”‚ โ”œโ”€โ”€ reconciler.py Every 60s: compare DB โ†” HL state, mark drift +โ”‚ โ”œโ”€โ”€ circuit_breaker.py Per-system (sys1/sys2) CB, daily DD + N-loss streak +โ”‚ โ”œโ”€โ”€ signal_categories.py CRITICAL CONFIG โ€” sys1/sys2 sources, ladders, +โ”‚ โ”‚ leverage clamping, protective stop formulas +โ”‚ โ”œโ”€โ”€ regime_filter.py Sys1 only โ€” recent-move / vol-contraction gates +โ”‚ โ”œโ”€โ”€ analysis.py AI signal scoring (DeepSeek) for Trump posts +โ”‚ โ”œโ”€โ”€ x_analysis.py AI scoring for X (Twitter) KOL posts. Three tiers: +โ”‚ โ”‚ TRADE_SIGNAL / DIRECTIONAL / NOISE. Strict NOISE +โ”‚ โ”‚ default โ€” most X posts should be filtered out. +โ”‚ โ”‚ Consumed by kol_x.py. tickers come out in the +โ”‚ โ”‚ {ticker,action,conviction} shape kol_divergence reads. +โ”‚ โ”œโ”€โ”€ x_poster.py X (Twitter) auto-poster for Trump signals. +โ”‚ โ”‚ Fires a prediction tweet then a follow-up at +โ”‚ โ”‚ +x_followup_minutes with the actual move. Gated by +โ”‚ โ”‚ x_enabled env var (False by default). Full no-op +โ”‚ โ”‚ if creds missing โ€” never blocks signal flow. +โ”‚ โ”œโ”€โ”€ entry_filter.py Cheap text-based pre-filter (skip RT/URL-only) +โ”‚ โ”œโ”€โ”€ telegram.py send_message / edit_message / answer_callback +โ”‚ โ”œโ”€โ”€ telegram_bot.py Long-poll loop + /start /digest /adopt /release ... +โ”‚ โ”œโ”€โ”€ telegram_digest.py Daily 3-section brief (rule-based; no LLM) +โ”‚ โ”œโ”€โ”€ price_store.py In-memory latest price per asset +โ”‚ โ”œโ”€โ”€ price_backfill.py Backfill historical 5min bars from Binance +โ”‚ โ”œโ”€โ”€ hl_price_feed.py Supplemental HL price feed for HL-native assets +โ”‚ โ”‚ (HYPE, PURR). Polls allMids every 2s. Runs +โ”‚ โ”‚ alongside binance.py. Without this, TP/SL +โ”‚ โ”‚ silently stops protecting HL-native trades. +โ”‚ โ”œโ”€โ”€ backtest.py Single-post backtest harness. Fetches 1m Binance +โ”‚ โ”‚ candles for [published_at, +max_hold_h] and +โ”‚ โ”‚ replays current exit rules. Conservative (uses +โ”‚ โ”‚ HIGH/LOW within bar). No fees. Batch runner on top. +โ”‚ โ”œโ”€โ”€ crypto.py HL API-key envelope encryption. enc:v2 = +โ”‚ โ”‚ PBKDF2-salted (H4 fix); enc:v1 read-compat. +โ”‚ โ”‚ scripts/reencrypt_keys.py upgrades stored rows. +โ”‚ โ”œโ”€โ”€ scanner_state.py In-memory toggle + observability for scanners +โ”‚ โ”œโ”€โ”€ macro/ +โ”‚ โ”‚ โ”œโ”€โ”€ fetchers.py 8 macro indicator HTTP fetchers (each @_none_on_fail) +โ”‚ โ”‚ โ”œโ”€โ”€ scoring.py Weighted composite -100..+100 +โ”‚ โ”‚ โ””โ”€โ”€ poll.py Daily UPSERT into macro_snapshots +โ”‚ โ”œโ”€โ”€ scanners/ +โ”‚ โ”‚ โ”œโ”€โ”€ btc_bottom_reversal.py 2-of-3 AHR999 + 200WMA + Pi Bottom +โ”‚ โ”‚ โ”œโ”€โ”€ funding_reversal.py Hourly funding extreme +โ”‚ โ”‚ โ””โ”€โ”€ sma_reclaim.py (archive โ€” not scheduled) +โ”‚ โ”œโ”€โ”€ kol_substack.py RSS ingest for 29 KOL feeds (substack/blog/podcast) +โ”‚ โ”œโ”€โ”€ kol_x.py X (Twitter) ingest via twitterapi.io โ†’ x_analysis โ†’ +โ”‚ โ”‚ KolPost(source="twitter"). Daily 01:30 UTC. No-op +โ”‚ โ”‚ if twitterapi_io_key unset. Provides the post-side +โ”‚ โ”‚ feed for X-only KOLs (andrewkang, murad). +โ”‚ โ”œโ”€โ”€ kol_onchain.py HL public API + Etherscan diff +โ”‚ โ”œโ”€โ”€ kol_divergence.py Cross-ref talks vs trades within ยฑ7d +โ”‚ โ”œโ”€โ”€ kol_analysis.py AI ticker/direction/conviction extract (Substack). +โ”‚ โ”‚ `_derive_tier()` maps its conviction + talks-vs- +โ”‚ โ”‚ trades score โ†’ the SAME trade_signal/directional/ +โ”‚ โ”‚ noise tiers x_analysis emits, so non-Twitter posts +โ”‚ โ”‚ get tier set in kol_substack (SIGNAL/VIEW badges + +โ”‚ โ”‚ "Signals only" filter work for blog/substack/pod). +โ”‚ โ”œโ”€โ”€ bottom_indicators.py AHR999 / Pi Cycle / 200WMA math +โ”‚ โ”œโ”€โ”€ funding_signal.py Real-time funding extreme detector +โ”‚ โ”œโ”€โ”€ signed_request.py EIP-191 signature verification (+ replay cache). +โ”‚ โ”‚ signed_read_creds / optional_signed_read_creds: +โ”‚ โ”‚ header-based (X-Sig-Ts/X-Sig-Sig) creds for read +โ”‚ โ”‚ endpoints, query fallback deprecated (C3 fix). +โ”‚ โ””โ”€โ”€ http_client.py Shared pooled httpx.AsyncClient (keep-alive). +โ”‚ Hot paths (scrapers, telegram send/poll, +โ”‚ hl_price_feed, binance REST, x_poster) use +โ”‚ get_client() with per-request timeout instead of +โ”‚ new-client-per-call. Closed in lifespan shutdown. +โ”œโ”€โ”€ scrapers/ +โ”‚ โ”œโ”€โ”€ truth_social.py CNN archive poller (5s interval) โ€” primary. +โ”‚ โ”‚ Conditional GET (ETag/Last-Modified โ†’ 304 skips +โ”‚ โ”‚ the 30k-post download), batch dedup (1 IN-query +โ”‚ โ”‚ per poll instead of 50 SELECTs), per-post +โ”‚ โ”‚ commit+dispatch so an actionable post never +โ”‚ โ”‚ waits behind older entries' AI analysis. +โ”‚ โ”‚ dispatch_post() is the shared WS/TG/X/trade +โ”‚ โ”‚ fan-out used by BOTH pollers. +โ”‚ โ””โ”€โ”€ trumpstruth.py trumpstruth.org RSS fallback poller. Same post id +โ”‚ hash โ†’ automatic dedup. Whoever sees first wins. +โ”‚ Offset by half the interval so the two pollers +โ”‚ don't hit upstream simultaneously. +โ”œโ”€โ”€ ws/ +โ”‚ โ””โ”€โ”€ manager.py WebSocket fan-out for live UI updates. +โ”‚ Broadcasts trade_alert events (execution_failed / +โ”‚ insufficient_balance / budget_reached) via +โ”‚ _broadcast_trade_alert() in bot_engine.py. +โ”œโ”€โ”€ models.py โ˜… All SQLAlchemy models in one file +โ”œโ”€โ”€ database.py Async engine + session factory +โ”œโ”€โ”€ config.py Pydantic Settings โ€” reads .env +โ””โ”€โ”€ main.py FastAPI lifespan, scheduler setup, route mount + Includes: singleton lock guard (one-leader, multi- + worker safe), deep health check /api/health/deep, + boot-grace window for price feeds. + +alembic/versions/ Migrations (numbered NNN). Latest = 026 + 026 = composite index (wallet_address, closed_at) on bot_trades +scripts/ One-shot ops +โ”œโ”€โ”€ preflight.py Pre-launch readiness gate (env / DB / TG / AI) +โ”œโ”€โ”€ launch_smoke.py End-to-end smoke (14 checks against running API) +โ”œโ”€โ”€ launch_seed.py Pre-launch data prep: drops test sources, trims KOL +โ”‚ window to last 30d, refetches all upstream sources. +โ”‚ Run ONCE before flipping traffic to a fresh DB. +โ”œโ”€โ”€ seed_kol_wallets.py Seeds the KOL wallet table with known addresses. +โ”‚ Idempotent (INSERT OR IGNORE). Run once at first deploy. +โ”œโ”€โ”€ rescore_v5.py Re-score every Post with current AI prompt +โ”œโ”€โ”€ backfill_signals.py Fill in signal for posts missing it +โ”œโ”€โ”€ reencrypt_keys.py Upgrade stored HL keys to enc:v2 (+KEK rotation +โ”‚ via OLD_ENCRYPTION_KEY). Idempotent; --dry-run. +โ””โ”€โ”€ verify_sys2_lifecycle.py Manual System-2 lifecycle walk-through + +tests/ pytest, 112 tests, fast (<3s total) +โ”œโ”€โ”€ test_adoption.py Adoption + release flow (snapshot-style, no real HL/AI) +โ”œโ”€โ”€ test_telegram_digest.py Daily digest formatting +โ”œโ”€โ”€ test_kol_tier.py kol_analysis._derive_tier (non-Twitter tier mapping) +โ”œโ”€โ”€ test_kol_x.py X ingest: dedup / mapping / no-op (mocked fetch+AI) +โ”œโ”€โ”€ test_ratelimit.py Rate limit coverage (BUG-02 fix) +โ”œโ”€โ”€ test_bottom_reversal_strategy.py btc_bottom_reversal 2-of-3 logic +โ”œโ”€โ”€ test_macro_fetchers_timing.py Macro fetcher timeout + error handling +โ””โ”€โ”€ test_production_readiness.py Environment / config sanity checks +``` + +--- + +## The two trading systems (memorise this) + +``` + System 1 System 2 + โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +Source "truth" "btc_bottom_reversal" + (+"funding_reversal" alert-only) +Trigger Trump posts a thing Daily scanner: 2-of-3 confluence +Latency need Seconds (price moves fast) Days/weeks (signal lives a long time) +Open path Auto (bot_engine.process_post fires _execute_for_subscriber) + MANUAL: user opens on HL UI, then + /adopt hands it to the bot +Stop loss User-configured + tight sys2_protective_stop_pct(actual_lev) + 1.5% floor (TRUMP_*) = 85% ร— (100/lev), capped at 35% +Exit model TP / trailing / SL 5-rung stop ladder + downside de-risk + + pyramid + peak-trail. NO TP. +Min hold 30 min (suppresses TP) n/a +Max hold 6h 18 months (ladder is the real exit) +Sizing base ร— regime multiplier Whatever user opened on HL +Concurrency cap n/a 3 positions / wallet (correlated beta) +Confidence min 88 (platform) / user 85 (platform) +Circuit breaker sys1_* sys2_* (independent) +Daily budget Full daily_budget_usd n/a โ€” user controls notional on HL +Telegram alert Trump alert format Macro/funding alert + /adopt CTA +X tweet Optional prediction tweet n/a +``` + +**If you're tempted to put sys2 logic in `_execute_for_subscriber`**: stop. +`process_post()` early-returns for sys2. The function only runs for sys1 now. +The dead sys2 branches inside `_execute_for_subscriber` are kept for diff +minimalism โ€” don't extend them. + +--- + +## The /adopt flow (System-2 lifecycle in detail) + +``` +1. Scanner fires + โ””โ”€ POST /api/signals/ingest (source=btc_bottom_reversal, signal=buy) + โ””โ”€ Post row created + โ””โ”€ process_post() early-returns for sys2 (no auto-open) + โ””โ”€ notify_signal() โ†’ Telegram fan-out with /adopt CTA appended + +2. User opens BTC long on Hyperliquid manually + โ””โ”€ size / leverage of their choice + +3. User in bot: /adopt + โ””โ”€ adoption.list_hl_positions(wallet) reads HL state + โ””โ”€ Telegram inline keyboard: tap [๐ŸŸข BTC long $1500 @72k ยท 2x] + โ””โ”€ Mode picker: [๐Ÿ“ˆ Standard] or [๐Ÿš€ Aggressive] + โ””โ”€ adoption.adopt_position(wallet, asset, mode): + a. Per-wallet asyncio lock acquired + b. Pre-flight: no_subscription / no_hl_key / paper_mode / + macro_disabled (Subscription.macro_enabled must be ON โ€” /adopt is the + sys2 management entry point, so the Macro Vibes toggle gates it) / + circuit_breaker (sys2 CB still gates adopt!) / + already_adopted / concurrency_cap (3) + c. Re-read HL state inside lock (fresh entry/size/lev) + d. Reject if leverage > SYS2_MAX_LEVERAGE (BUG-09 fix) + e. Resolve sys2_protective_stop_pct(HL_actual_leverage) + f. INSERT BotTrade with eff_* frozen + sys2_mode + hl_order_id="adopted:" + + trigger_post_id=NULL + g. register_trade() with full ladder/de-risk/addon/peak_trail + +4. tp_sl_monitor drives the position + โ””โ”€ Stop ratchet, downside de-risk partial reduces, pyramid add-ons, + peak-trail close, max_hold backstop. All staged through the + lock-protected partial_derisk / pyramid_add / close_and_finalize. + +5a. User wants out: /release + โ””โ”€ Sets BotTrade.released_at = now; unregister(trade_id) from watchdog + โ””โ”€ HL position UNTOUCHED โ€” bot stops driving, user has manual control + +5b. Bot drives the close (ladder / max-hold) + โ””โ”€ close_and_finalize() atomic claim sets closed_at, computes pnl + +5c. User force-closes via UI: POST /api/positions/{id}/close + โ””โ”€ manual_close calls close_and_finalize(force=True) โ€” bypasses + released_at guard. Works on adopted-and-released trades too. + +6. Recovery on restart: + โ””โ”€ recovery.rehydrate_open_trades reads BotTrade WHERE closed_at IS NULL + AND released_at IS NULL + โ””โ”€ For each sys2 trade: rebuild ladder from sys2_mode + adopted fallback. + Also reschedules _time_stop_check with remaining seconds (elapsed windows + fire immediately with delay=0). This is the critical fix โ€” without it + adopted trades lose their entire sys2 ladder on restart. +``` + +--- + +## Singleton lock guard (critical infrastructure detail) + +`main.py:lifespan` calls `_acquire_singleton_lock()` at boot using an +advisory file lock (`/tmp/trumpsignal-backend.lock`, configurable via +`SINGLETON_LOCK_PATH`). Only ONE process โ€” "the leader" โ€” starts the +scheduler, scrapers, price feeds, and Telegram poller. Any additional +worker (e.g. accidental `--workers 2`) serves HTTP reads only and logs +`SINGLETON LOCK NOT ACQUIRED`. The OS releases the lock automatically +when the leader exits, so a crashed leader unblocks the next start. + +The `/api/health/deep` endpoint surfaces `"is_leader": false` and adds +it to `problems[]` so uptime monitors catch mis-configured multi-worker +deployments. + +--- + +## Critical invariants checklist (when reviewing any trading change) + +- [ ] Does this code path respect `released_at IS NULL`? +- [ ] Does it use `eff_*` (frozen) not live `Subscription.*` for exit math? +- [ ] If it opens a new position, does it use HL's actual leverage (not requested)? +- [ ] If it touches an open position concurrently, is it wrapped in the + per-trade `_lock_for(trade_id)` lock? +- [ ] If it opens, is it inside `_wallet_lock(wallet)` so the budget / + concurrency check is atomic with the write? +- [ ] If it closes, does it use the conditional `UPDATE ... WHERE closed_at + IS NULL` atomic claim? +- [ ] Does it handle the `already_closed` path from HL gracefully (preserve + banked partial PnL)? +- [ ] Does it correctly check sys1 vs sys2 CB independently? + +--- + +## Common workflows + +### Add a new signal source + +1. Decide: System 1 (auto-trade) or System 2 (alert + /adopt) or alert-only? +2. Write a scanner under `app/services/scanners/NEW.py` that posts to + `POST /api/signals/ingest` with `{source: "your_new_source", ...}`. +3. Schedule it in `app/main.py` (`_scheduler.add_job`). +4. Add the source to `signal_categories.SYSTEM_1_SOURCES` or + `SYSTEM_2_SOURCES` if it should trade. Leave it out if alert-only. +5. Add a Telegram preference column to `TelegramBinding` (migration) + + a mapping entry in `telegram._pref_column_for_source`. +6. Add a label to `telegram._source_label` and `_signal_emoji`. +7. Add a `/yoursource on|off` command in `telegram_bot.py`. +8. If sys2: extend `signal_categories._CATEGORY_EXITS` if it needs a custom + exit profile (otherwise default works). +9. Add the deep-link path in `telegram.format_post` AND `telegram.format_public_post`. + +### Add a new bot command + +1. `_cmd_x` async function in `telegram_bot.py`. +2. Route it in `_handle_message`. +3. If it needs inline buttons: build `reply_markup` payload, handle + callbacks in `_handle_callback` (route by `callback_data` prefix). +4. Update `HELP_TEXT` and remind the user to add it to BotFather + `/setcommands` after deploy. + +### Add a column to an existing table + +1. New migration `alembic/versions/NNN_description.py`. + - Use `op.batch_alter_table` (sqlite-compatible). + - Default values via `server_default=` so backfill is implicit. +2. Mirror the field on the SQLAlchemy model in `app/models.py`. +3. Apply locally: `DATABASE_URL= alembic upgrade head`. + +### Enable X (Twitter) posting + +1. Create a Twitter developer app with OAuth 1.0a user context permissions. +2. Set in `.env`: `X_API_KEY`, `X_API_SECRET`, `X_ACCESS_TOKEN`, + `X_ACCESS_SECRET`, `X_ENABLED=true`. +3. Optionally tune: `X_DAILY_CAP` (default 40), `X_FOLLOWUP_MINUTES` (default 15). +4. Verify with a dry-run: set `X_ENABLED=false` and check logs โ€” the poster + logs what it WOULD tweet without sending. + +### Deploy + +```bash +# On the server: +DATABASE_URL=$PROD_URL alembic upgrade head +systemctl restart trumpalpha-backend # or whatever the unit is +python scripts/preflight.py # MUST pass before flipping traffic +python scripts/launch_smoke.py --base https://api.trumpalpha.io +``` + +--- + +## Running it + +```bash +cd backend && python -m venv venv && source venv/bin/activate +pip install -r requirements.txt +cp .env.example .env # fill in: ENCRYPTION_KEY, AI_API_KEY, INGEST_API_KEY +uvicorn app.main:app --reload # dev only; prod uses --workers 1, no --reload +``` + +--- + +## Testing + +```bash +source venv/bin/activate +python -m pytest tests/ -q # 112 tests, ~2s +python scripts/preflight.py # env + DB + TG + AI auth checks +python scripts/launch_smoke.py # 14 end-to-end checks vs running API +``` + +Adoption + telegram_digest are snapshot-style (no real HL/AI). +End-to-end trading is verified manually via the bot. + +--- + +## Telegram bot mechanics (since it's a custom integration) + +- **Long-poll mode** via `getUpdates`. Only ONE process can long-poll a + given bot token at a time โ€” if you horizontally scale, switch to + webhook (not done yet). +- Bot must be re-bound via `@BotFather` `/setcommands` whenever new + commands are added (the slash-menu users see is separate from what + the bot internally handles). +- `send_message` returns `False` on failure; per-user binding rows track + `total_alerts_sent` / `total_alerts_failed` counters. +- **Inline keyboards** = the `reply_markup` payload to `sendMessage`. + Callback data is capped at 64 bytes; keep it short (`adopt:mode:BTC:standard`). + `_handle_callback` MUST end with `answer_callback` or the button spins + forever on the user's client. +- Free tier = walletless `/start` (chat_id only). Pro tier = wallet bound + via `/start CODE` where CODE comes from Settings UI. + +--- + +## Why "Macro Vibes" became manage-only (the ADR) + +V1.0: System 2 auto-opened sys2 trades on user wallets. Carried real +execution surface: leverage clipping, daily budget split, concurrency caps, +sys2 paper branches, key handling per user. Audit surfaced ~6 bugs. + +V2.0 (current): sys2 manage-only. The strategy is day-K โ€” entry delay of +24h doesn't matter. The valuable part is multi-month exit management +(5-rung ladder, de-risk, pyramid, peak-trail), which still runs against +positions the user adopts. + +**Net effect**: massive reduction in execution risk surface; same alpha +(strategy logic unchanged); legal/responsibility shifts from "bot opened +this for you" to "you opened it, bot manages your discipline". + +--- + +## Things that LOOK like bugs but aren't + +- **`_execute_for_subscriber` has lots of `if sub["_is_system_2"]` branches.** + Dead code under v2.0 (process_post early-returns for sys2). Kept for diff + minimalism โ€” don't extend or re-enable. +- **`kol_x.X_KOLS` handle โ‰  X username.** `handle` is the CANONICAL key + (e.g. "cryptohayes") that MUST match `KolWallet.handle` so divergence can + join post-side โ†” on-chain; `x_username` is the screen name fetched from X. + andrewkang/murad wallets stay dark (zero divergence detections) until X + ingestion supplies their post side โ€” that's the whole reason kol_x exists. + Empty `twitterapi_io_key` โ†’ kol_x is a full no-op (no error). +- **`funding_reversal` source is in `SYSTEM_2_SOURCES`? No.** It's + intentionally NOT in either supported set โ€” it ingests as a Post for + audit + sends Telegram alert via the CTA path, but doesn't trigger any + auto-trading. Adopt still works (it's asset-based, not signal-based). +- **`Subscription.sys2_budget_pct` defaults to 0.7.** Legacy field from the + auto-open era. With v2.0 manage-only, it's effectively unused โ€” sys1 + (Trump) reads full `daily_budget_usd`. Don't read it for new code. +- **Adopted trades have `hl_order_id` starting with `"adopted:"`.** Distinct + from auto-opened (HL order id integer) and paper (`"paper"` literal). + Useful for telemetry filtering โ€” AND it's the canonical sys2 marker the + daily-budget query uses, because adopted trades have `trigger_post_id=NULL` + so source-based classification fails (M1 fix). The `/trades` serializer also + reports `trigger_source="adopted"` from this prefix. +- **`/signals/accuracy` is scoped to production sources + buy/short.** It + intentionally restricts to `SUPPORTED_TRADING_SOURCES` and the CURRENT + buy/short vocabulary โ€” retired/test sources (rsi_reversal, sma_reclaim, + breakout, phase1, `test`) and the legacy `sell` signal are excluded so the + public accuracy scoreboard reflects what the live bot actually trades. +- **macro_enabled vs Telegram alerts are TWO separate switches.** Turning off + Macro Vibes (`Subscription.macro_enabled`) gates sys2 *management* โ€” it now + blocks `/adopt` (macro_disabled). It does NOT silence Telegram alerts; those + are controlled independently by the per-source `TelegramBinding` preference + columns. "Alerts on, don't auto-manage" is a deliberately supported combo. +- **`telegram.send_message` accepts `int | str` for `chat_id`.** Intentional. + Integer = private chat, string = public channel username (e.g. `"@trumpalpha"`). +- **`format_public_post` deliberately omits `expected_move_pct`, + `invalidation_price`, and `/adopt` CTA.** Execution-sensitive data stays + private. The public version shows confidence tier (HIGH/MED/LOW) instead + of the raw score. +- **`_adopt_locks` in adoption.py** is an `OrderedDict` capped at 512 with + LRU eviction โ€” matches the `_WALLET_LOCK_MAX` pattern in `bot_engine`. +- **`trumpstruth.py` runs at half-interval offset** โ€” the two CNN + trumpstruth + pollers are deliberately staggered so they don't hammer upstream simultaneously. + The offset is `truth_social_poll_seconds // 2`, set via APScheduler + `next_run_time` at boot. + +--- + +## Open known issues (not blocking launch but worth fixing later) + +- **`adopt:choose:BTC` callback may show stale prices** if user takes >60s + to tap (HL fees, partial fills can change entry/size). adopt_position + re-reads HL at mode-tap time so the FROZEN BotTrade is always fresh, + but the picker label could be outdated. + +- ~~**Telegram bot offset on restart**~~ **FIXED 2026-06-01**: startup drain + added. Stale `/adopt` replays suppressed. + +- **`/adopt` picker label** can show stale price if user waits >60s to tap + (frozen BotTrade is always fresh, but the Telegram picker label may be outdated). + +- ~~**M1 adopted positions miscounted against sys1 budget**~~ **FIXED 2026-06-09**: + the daily-budget query in `bot_engine` now treats any trade whose + `hl_order_id` starts with `"adopted:"` as sys2, regardless of the NULL + `trigger_post_id`. Previously the outerjoin to Post yielded src=NULL โ†’ + classified as sys1 โ†’ every adopted macro position inflated the Trump + scalp budget and could prematurely trip `budget_reached`. + +- **Funding-reversal `/adopt` uses the btc_bottom_reversal exit profile** + (`ADOPTED_CATEGORY` is fixed). This is BY DESIGN, not a bug: adopt is + ASSET-based (the user opens any position on HL and adopts it) โ€” the bot has + no reliable link back to which signal motivated it, and the sys2 ladder is + direction/horizon-agnostic. Changing this needs a source-tracking mechanism + at adopt time (an ADR), not a one-line tweak. + +**Deferred security items โ€” ALL RESOLVED 2026-06-12:** +- ~~**C3**~~ FIXED: signed reads now send `X-Sig-Ts` / `X-Sig-Sig` HEADERS + (see `signed_read_creds` in signed_request.py). Legacy `?ts=&sig=` query + params still accepted (deprecated) for old clients. +- ~~**H4**~~ FIXED: keys now encrypt as `enc:v2` (PBKDF2-HMAC-SHA256, per-blob + salt, 600k iters). v1 blobs still decrypt; run `scripts/reencrypt_keys.py` + ONCE in prod (after DB backup) to upgrade stored rows. +- ~~**M5**~~ Was already fixed: unauthenticated `/telegram/{wallet}/status` + returns only `configured`/`bound` booleans; full details require a signed read. + +--- + +## Repos in this project + +- **This repo** (`/Users/k/Public/trumpsignal/backend`) โ€” Python/FastAPI backend +- **Sibling frontend** (`/Users/k/Public/trumpsignal/frontend`) โ€” Next.js 16 + dashboard at trumpsignal.com. See its own AGENTS.md. + +Both deployed independently. Backend serves the JSON API + Telegram bot. +Frontend is a thin SPA over the API + WebSocket. diff --git a/CLAUDE.md b/CLAUDE.md index e7ae73b..7bcbbb7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -105,7 +105,13 @@ linked) = Trump auto-trade + /adopt manage-only flow for sys2. - **Trading**: Hyperliquid SDK; API-wallet keys are envelope-encrypted with `ENCRYPTION_KEY` (KEK), per-user DEK derivation via `crypto.py`. - **Prices**: Two feeds: - - `binance.py` WebSocket โ†’ BTC, ETH, SOL, TRUMP, BNB, DOGE, LINK, AAVE + - `binance.py` WebSocket โ†’ 30 mainstream perps (BTC, ETH, SOL, TRUMP, BNB, + DOGE, LINK, AAVE + AVAX/ARB/OP/SUI/APT/INJ/ATOM/XRP/LTC/ADA/MATIC/SHIB/ + PEPE/WIF/BONK/TAO/JUP/RENDER/FET/TIA/SEI/PENDLE). The WS URL is built FROM + `ASSET_MAP` so the two never drift. Any asset NOT in ASSET_MAP loses + TP/SL/trailing protection (only max-hold remains) โ€” see the ASSET_MAP + docstring. `tp_sl_monitor.register_trade` logs an ERROR if a trade opens on + an uncovered asset. - `hl_price_feed.py` polls HL `allMids` every 2s โ†’ HYPE, PURR (HL-native assets not on Binance) Both pump `price_store` + `tp_sl_monitor` on every tick. - **Telegram**: long-poll mode (single instance), HTML messages, inline @@ -173,7 +179,9 @@ app/ โ”‚ โ”‚ candles for [published_at, +max_hold_h] and โ”‚ โ”‚ replays current exit rules. Conservative (uses โ”‚ โ”‚ HIGH/LOW within bar). No fees. Batch runner on top. -โ”‚ โ”œโ”€โ”€ crypto.py HL API-key envelope encryption +โ”‚ โ”œโ”€โ”€ crypto.py HL API-key envelope encryption. enc:v2 = +โ”‚ โ”‚ PBKDF2-salted (H4 fix); enc:v1 read-compat. +โ”‚ โ”‚ scripts/reencrypt_keys.py upgrades stored rows. โ”‚ โ”œโ”€โ”€ scanner_state.py In-memory toggle + observability for scanners โ”‚ โ”œโ”€โ”€ macro/ โ”‚ โ”‚ โ”œโ”€โ”€ fetchers.py 8 macro indicator HTTP fetchers (each @_none_on_fail) @@ -190,12 +198,32 @@ app/ โ”‚ โ”‚ feed for X-only KOLs (andrewkang, murad). โ”‚ โ”œโ”€โ”€ kol_onchain.py HL public API + Etherscan diff โ”‚ โ”œโ”€โ”€ kol_divergence.py Cross-ref talks vs trades within ยฑ7d -โ”‚ โ”œโ”€โ”€ kol_analysis.py AI ticker/direction/conviction extract (Substack) +โ”‚ โ”œโ”€โ”€ kol_analysis.py AI ticker/direction/conviction extract (Substack). +โ”‚ โ”‚ `_derive_tier()` maps its conviction + talks-vs- +โ”‚ โ”‚ trades score โ†’ the SAME trade_signal/directional/ +โ”‚ โ”‚ noise tiers x_analysis emits, so non-Twitter posts +โ”‚ โ”‚ get tier set in kol_substack (SIGNAL/VIEW badges + +โ”‚ โ”‚ "Signals only" filter work for blog/substack/pod). โ”‚ โ”œโ”€โ”€ bottom_indicators.py AHR999 / Pi Cycle / 200WMA math โ”‚ โ”œโ”€โ”€ funding_signal.py Real-time funding extreme detector -โ”‚ โ””โ”€โ”€ signed_request.py EIP-191 signature verification (+ replay cache) +โ”‚ โ”œโ”€โ”€ signed_request.py EIP-191 signature verification (+ replay cache). +โ”‚ โ”‚ signed_read_creds / optional_signed_read_creds: +โ”‚ โ”‚ header-based (X-Sig-Ts/X-Sig-Sig) creds for read +โ”‚ โ”‚ endpoints, query fallback deprecated (C3 fix). +โ”‚ โ””โ”€โ”€ http_client.py Shared pooled httpx.AsyncClient (keep-alive). +โ”‚ Hot paths (scrapers, telegram send/poll, +โ”‚ hl_price_feed, binance REST, x_poster) use +โ”‚ get_client() with per-request timeout instead of +โ”‚ new-client-per-call. Closed in lifespan shutdown. โ”œโ”€โ”€ scrapers/ -โ”‚ โ”œโ”€โ”€ truth_social.py CNN archive poller (5s interval) โ€” primary +โ”‚ โ”œโ”€โ”€ truth_social.py CNN archive poller (5s interval) โ€” primary. +โ”‚ โ”‚ Conditional GET (ETag/Last-Modified โ†’ 304 skips +โ”‚ โ”‚ the 30k-post download), batch dedup (1 IN-query +โ”‚ โ”‚ per poll instead of 50 SELECTs), per-post +โ”‚ โ”‚ commit+dispatch so an actionable post never +โ”‚ โ”‚ waits behind older entries' AI analysis. +โ”‚ โ”‚ dispatch_post() is the shared WS/TG/X/trade +โ”‚ โ”‚ fan-out used by BOTH pollers. โ”‚ โ””โ”€โ”€ trumpstruth.py trumpstruth.org RSS fallback poller. Same post id โ”‚ hash โ†’ automatic dedup. Whoever sees first wins. โ”‚ Offset by half the interval so the two pollers @@ -225,11 +253,14 @@ scripts/ One-shot ops โ”‚ Idempotent (INSERT OR IGNORE). Run once at first deploy. โ”œโ”€โ”€ rescore_v5.py Re-score every Post with current AI prompt โ”œโ”€โ”€ backfill_signals.py Fill in signal for posts missing it +โ”œโ”€โ”€ reencrypt_keys.py Upgrade stored HL keys to enc:v2 (+KEK rotation +โ”‚ via OLD_ENCRYPTION_KEY). Idempotent; --dry-run. โ””โ”€โ”€ verify_sys2_lifecycle.py Manual System-2 lifecycle walk-through -tests/ pytest, 83 tests, fast (<3s total) +tests/ pytest, 112 tests, fast (<3s total) โ”œโ”€โ”€ test_adoption.py Adoption + release flow (snapshot-style, no real HL/AI) โ”œโ”€โ”€ test_telegram_digest.py Daily digest formatting +โ”œโ”€โ”€ test_kol_tier.py kol_analysis._derive_tier (non-Twitter tier mapping) โ”œโ”€โ”€ test_kol_x.py X ingest: dedup / mapping / no-op (mocked fetch+AI) โ”œโ”€โ”€ test_ratelimit.py Rate limit coverage (BUG-02 fix) โ”œโ”€โ”€ test_bottom_reversal_strategy.py btc_bottom_reversal 2-of-3 logic @@ -292,6 +323,8 @@ minimalism โ€” don't extend them. โ””โ”€ adoption.adopt_position(wallet, asset, mode): a. Per-wallet asyncio lock acquired b. Pre-flight: no_subscription / no_hl_key / paper_mode / + macro_disabled (Subscription.macro_enabled must be ON โ€” /adopt is the + sys2 management entry point, so the Macro Vibes toggle gates it) / circuit_breaker (sys2 CB still gates adopt!) / already_adopted / concurrency_cap (3) c. Re-read HL state inside lock (fresh entry/size/lev) @@ -432,7 +465,7 @@ uvicorn app.main:app --reload # dev only; prod uses --workers 1, no --reload ```bash source venv/bin/activate -python -m pytest tests/ -q # 79 tests, ~2s +python -m pytest tests/ -q # 112 tests, ~2s python scripts/preflight.py # env + DB + TG + AI auth checks python scripts/launch_smoke.py # 14 end-to-end checks vs running API ``` @@ -498,7 +531,20 @@ this for you" to "you opened it, bot manages your discipline". (Trump) reads full `daily_budget_usd`. Don't read it for new code. - **Adopted trades have `hl_order_id` starting with `"adopted:"`.** Distinct from auto-opened (HL order id integer) and paper (`"paper"` literal). - Useful for telemetry filtering. + Useful for telemetry filtering โ€” AND it's the canonical sys2 marker the + daily-budget query uses, because adopted trades have `trigger_post_id=NULL` + so source-based classification fails (M1 fix). The `/trades` serializer also + reports `trigger_source="adopted"` from this prefix. +- **`/signals/accuracy` is scoped to production sources + buy/short.** It + intentionally restricts to `SUPPORTED_TRADING_SOURCES` and the CURRENT + buy/short vocabulary โ€” retired/test sources (rsi_reversal, sma_reclaim, + breakout, phase1, `test`) and the legacy `sell` signal are excluded so the + public accuracy scoreboard reflects what the live bot actually trades. +- **macro_enabled vs Telegram alerts are TWO separate switches.** Turning off + Macro Vibes (`Subscription.macro_enabled`) gates sys2 *management* โ€” it now + blocks `/adopt` (macro_disabled). It does NOT silence Telegram alerts; those + are controlled independently by the per-source `TelegramBinding` preference + columns. "Alerts on, don't auto-manage" is a deliberately supported combo. - **`telegram.send_message` accepts `int | str` for `chat_id`.** Intentional. Integer = private chat, string = public channel username (e.g. `"@trumpalpha"`). - **`format_public_post` deliberately omits `expected_move_pct`, @@ -527,11 +573,29 @@ this for you" to "you opened it, bot manages your discipline". - **`/adopt` picker label** can show stale price if user waits >60s to tap (frozen BotTrade is always fresh, but the Telegram picker label may be outdated). -**Deferred security items (need interface change or DB migration):** -- **C3**: Read endpoints pass `ts`/`sig` as URL query params โ†’ access log exposure. -- **H4**: HL API key KEK uses single unsalted SHA-256. Needs re-encryption migration. -- **M1**: Adopted sys2 positions miscounted against sys1 daily budget. -- **M5**: `GET /telegram/{wallet}/status` unauthenticated โ€” exposes chat_id/tg_username. +- ~~**M1 adopted positions miscounted against sys1 budget**~~ **FIXED 2026-06-09**: + the daily-budget query in `bot_engine` now treats any trade whose + `hl_order_id` starts with `"adopted:"` as sys2, regardless of the NULL + `trigger_post_id`. Previously the outerjoin to Post yielded src=NULL โ†’ + classified as sys1 โ†’ every adopted macro position inflated the Trump + scalp budget and could prematurely trip `budget_reached`. + +- **Funding-reversal `/adopt` uses the btc_bottom_reversal exit profile** + (`ADOPTED_CATEGORY` is fixed). This is BY DESIGN, not a bug: adopt is + ASSET-based (the user opens any position on HL and adopts it) โ€” the bot has + no reliable link back to which signal motivated it, and the sys2 ladder is + direction/horizon-agnostic. Changing this needs a source-tracking mechanism + at adopt time (an ADR), not a one-line tweak. + +**Deferred security items โ€” ALL RESOLVED 2026-06-12:** +- ~~**C3**~~ FIXED: signed reads now send `X-Sig-Ts` / `X-Sig-Sig` HEADERS + (see `signed_read_creds` in signed_request.py). Legacy `?ts=&sig=` query + params still accepted (deprecated) for old clients. +- ~~**H4**~~ FIXED: keys now encrypt as `enc:v2` (PBKDF2-HMAC-SHA256, per-blob + salt, 600k iters). v1 blobs still decrypt; run `scripts/reencrypt_keys.py` + ONCE in prod (after DB backup) to upgrade stored rows. +- ~~**M5**~~ Was already fixed: unauthenticated `/telegram/{wallet}/status` + returns only `configured`/`bound` booleans; full details require a signed read. --- diff --git a/app/api/performance.py b/app/api/performance.py index ff6d45f..7adea80 100644 --- a/app/api/performance.py +++ b/app/api/performance.py @@ -8,7 +8,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.database import get_db from app.models import BotTrade from app.schemas import BotPerformance -from app.services.signed_request import verify_signed_request_any +from app.services.signed_request import ( + SignedReadCreds, signed_read_creds, verify_signed_request_any) router = APIRouter() logger = logging.getLogger(__name__) @@ -21,8 +22,7 @@ ACTION_VIEW_USER = "view_user" @router.get("/performance", response_model=BotPerformance) async def get_performance( wallet: str = Query(..., description="Wallet address (lower-cased internally)"), - ts: int = Query(..., description="Signed timestamp (ms)"), - sig: str = Query(..., description="EIP-191 signature"), + creds: SignedReadCreds = Depends(signed_read_creds), include_paper: bool = Query( False, description="Include paper (simulated) trades. Default false โ€” this " @@ -35,8 +35,8 @@ async def get_performance( verify_signed_request_any( actions=[ACTION_VIEW_PERFORMANCE, ACTION_VIEW_USER], wallet=wallet, - timestamp_ms=ts, - signature=sig, + timestamp_ms=creds.ts, + signature=creds.sig, body=None, allow_replay=True, ) diff --git a/app/api/positions.py b/app/api/positions.py index 8b3322a..d3ab876 100644 --- a/app/api/positions.py +++ b/app/api/positions.py @@ -35,7 +35,9 @@ from app.database import get_db from app.models import BotTrade, Subscription, iso_utc from app.services.crypto import decrypt_api_key from app.services.price_store import price_store -from app.services.signed_request import verify_signed_request, verify_signed_request_any +from app.services.signed_request import ( + SignedReadCreds, signed_read_creds, + verify_signed_request, verify_signed_request_any) router = APIRouter() logger = logging.getLogger(__name__) @@ -144,8 +146,7 @@ def _enrich(trade: BotTrade) -> OpenPosition: @router.get("/positions/open", response_model=OpenPositionsResponse) async def get_open_positions( wallet: str = Query(..., description="Wallet address (lower-cased internally)"), - ts: int = Query(..., description="Signed timestamp (ms)"), - sig: str = Query(..., description="EIP-191 signature"), + creds: SignedReadCreds = Depends(signed_read_creds), db: AsyncSession = Depends(get_db), ): """Live open positions for the wallet, with mark-to-market PnL.""" @@ -153,8 +154,8 @@ async def get_open_positions( verify_signed_request_any( actions=[ACTION_VIEW_POSITIONS, ACTION_VIEW_USER], wallet=wallet, - timestamp_ms=ts, - signature=sig, + timestamp_ms=creds.ts, + signature=creds.sig, body=None, allow_replay=True, ) @@ -177,8 +178,7 @@ async def get_open_positions( @router.get("/positions/today", response_model=TodayStatsResponse) async def get_today_stats( wallet: str = Query(...), - ts: int = Query(..., description="Signed timestamp (ms)"), - sig: str = Query(..., description="EIP-191 signature"), + creds: SignedReadCreds = Depends(signed_read_creds), db: AsyncSession = Depends(get_db), ): """Today's realized P&L (since UTC midnight) + open count. @@ -190,8 +190,8 @@ async def get_today_stats( verify_signed_request_any( actions=[ACTION_VIEW_POSITIONS, ACTION_VIEW_USER], wallet=wallet, - timestamp_ms=ts, - signature=sig, + timestamp_ms=creds.ts, + signature=creds.sig, body=None, allow_replay=True, ) @@ -453,8 +453,7 @@ class HLPositionsResponse(BaseModel): @router.get("/positions/hl/{wallet}", response_model=HLPositionsResponse) async def list_hl_positions_endpoint( wallet: str, - ts: int = Query(..., description="Signed timestamp (ms)"), - sig: str = Query(..., description="EIP-191 signature"), + creds: SignedReadCreds = Depends(signed_read_creds), ): """Read the wallet's CURRENT Hyperliquid open positions, annotated with 'already adopted' flag. Used by the Adopt picker on the frontend. @@ -469,8 +468,8 @@ async def list_hl_positions_endpoint( verify_signed_request_any( actions=[ACTION_VIEW_POSITIONS, ACTION_VIEW_USER], wallet=wallet, - timestamp_ms=ts, - signature=sig, + timestamp_ms=creds.ts, + signature=creds.sig, body=None, allow_replay=True, ) diff --git a/app/api/prices.py b/app/api/prices.py index 9069e9e..c5a383d 100644 --- a/app/api/prices.py +++ b/app/api/prices.py @@ -1,7 +1,6 @@ import logging from typing import List -import httpx from fastapi import APIRouter, HTTPException, Query, Request from fastapi.responses import Response @@ -33,9 +32,9 @@ async def fetch_binance_candles(asset: str, tf: str, limit: int) -> List[Candle] symbol = SYMBOL_MAP[asset] interval = BINANCE_INTERVAL[tf] url = f"{settings.binance_rest_url}/api/v3/klines?symbol={symbol}&interval={interval}&limit={limit}" - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.get(url) - resp.raise_for_status() + from app.services.http_client import get_client + resp = await get_client().get(url, timeout=15) + resp.raise_for_status() rows = resp.json() return [ Candle( diff --git a/app/api/telegram.py b/app/api/telegram.py index c4d0c11..ec975e1 100644 --- a/app/api/telegram.py +++ b/app/api/telegram.py @@ -25,7 +25,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.config import settings from app.database import get_db from app.models import TelegramBinding, Subscription -from app.services.signed_request import verify_signed_request +from app.services.signed_request import ( + SignedReadCreds, optional_signed_read_creds, verify_signed_request) from app.services.telegram import send_test_message from app.services.telegram_bot import issue_binding_code, unbind_wallet @@ -144,8 +145,7 @@ async def _build_status( @router.get("/{wallet}/status", response_model=StatusResponse) async def status( wallet: str, - timestamp: Optional[int] = None, - signature: Optional[str] = None, + creds: Optional[SignedReadCreds] = Depends(optional_signed_read_creds), db: AsyncSession = Depends(get_db), ) -> StatusResponse: """Wallet Telegram status. @@ -162,14 +162,14 @@ async def status( # Accepting view_user avoids requiring a separate wallet signature just to # see Telegram status โ€” the frontend reuses its cached view_user envelope. authenticated = False - if timestamp is not None and signature: + if creds is not None: for _action in ("view_telegram_status", "view_user"): try: verify_signed_request( action=_action, wallet=wallet, - timestamp_ms=timestamp, - signature=signature, + timestamp_ms=creds.ts, + signature=creds.sig, body=None, allow_replay=True, ) diff --git a/app/api/trades.py b/app/api/trades.py index 398463a..ef2ea33 100644 --- a/app/api/trades.py +++ b/app/api/trades.py @@ -9,7 +9,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.database import get_db from app.models import BotTrade, iso_utc from app.schemas import BotTrade as BotTradeSchema -from app.services.signed_request import verify_signed_request_any +from app.services.signed_request import ( + SignedReadCreds, signed_read_creds, verify_signed_request_any) router = APIRouter() logger = logging.getLogger(__name__) @@ -55,8 +56,7 @@ def _trade_to_schema(trade: BotTrade) -> BotTradeSchema: @router.get("/trades", response_model=List[BotTradeSchema]) async def get_trades( wallet: str = Query(..., description="Wallet address (lower-cased internally)"), - ts: int = Query(..., description="Signed timestamp (ms)"), - sig: str = Query(..., description="EIP-191 signature"), + creds: SignedReadCreds = Depends(signed_read_creds), limit: int = Query(default=20, ge=1, le=500), # raised from 100: Analytics needs 500 for full history page: int = Query(default=1, ge=1), db: AsyncSession = Depends(get_db), @@ -65,8 +65,8 @@ async def get_trades( verify_signed_request_any( actions=[ACTION_VIEW_TRADES, ACTION_VIEW_USER], wallet=wallet, - timestamp_ms=ts, - signature=sig, + timestamp_ms=creds.ts, + signature=creds.sig, body=None, allow_replay=True, ) diff --git a/app/api/user.py b/app/api/user.py index e3156c7..cd28082 100644 --- a/app/api/user.py +++ b/app/api/user.py @@ -16,7 +16,8 @@ from app.schemas import ( ) from app.services.crypto import encrypt_api_key from app.services.hyperliquid import HyperliquidTrader -from app.services.signed_request import verify_signed_request +from app.services.signed_request import ( + SignedReadCreds, signed_read_creds, verify_signed_request) router = APIRouter() logger = logging.getLogger(__name__) @@ -132,8 +133,7 @@ async def get_user_public(wallet: str, db: AsyncSession = Depends(get_db)): @router.get("/user/{wallet}", response_model=UserResponse) async def get_user( wallet: str, - ts: int = Query(..., description="Signed timestamp (ms)"), - sig: str = Query(..., description="EIP-191 signature"), + creds: SignedReadCreds = Depends(signed_read_creds), db: AsyncSession = Depends(get_db), ): wallet = wallet.lower().strip() @@ -143,8 +143,8 @@ async def get_user( verify_signed_request( action=ACTION_VIEW_USER, wallet=wallet, - timestamp_ms=ts, - signature=sig, + timestamp_ms=creds.ts, + signature=creds.sig, body=None, allow_replay=True, # idempotent GET โ€” allow sessionStorage caching for UX ) diff --git a/app/main.py b/app/main.py index a4dd6db..d8e08e3 100644 --- a/app/main.py +++ b/app/main.py @@ -139,17 +139,12 @@ async def lifespan(app: FastAPI): # 3. Start Truth Social poller via APScheduler _scheduler = AsyncIOScheduler() - # Signal monitor โ€” polls every 5 minutes - from app.services.funding_signal import poll_funding_signal - _scheduler.add_job( - poll_funding_signal, - "interval", - minutes=5, - id="funding_signal_poll", - max_instances=1, - coalesce=True, - ) - logger.info("Breakout signal monitor scheduled every 5 minutes.") + # Breakout signal monitor (poll_funding_signal, ETH/LINK 5m) UNSCHEDULED + # 2026-06-12: the feature has been disabled for months (_enabled=False, + # operator-only toggle) and the frontend panel that displayed it was + # removed โ€” the 5-min Binance kline poll was pure waste. The /signal/* + # API routes still work; to revive, re-add the add_job() here AND remount + # SignalMonitor in the frontend's MacroVibesPageClient. _scheduler.add_job( poll_truth_social, @@ -347,6 +342,8 @@ async def lifespan(app: FastAPI): await _telegram_task except asyncio.CancelledError: pass + from app.services.http_client import aclose as http_client_aclose + await http_client_aclose() await engine.dispose() logger.info("Shutdown complete.") diff --git a/app/scrapers/trumpstruth.py b/app/scrapers/trumpstruth.py index 3ebfdac..abdaa2f 100644 --- a/app/scrapers/trumpstruth.py +++ b/app/scrapers/trumpstruth.py @@ -24,10 +24,12 @@ from datetime import datetime, timezone from email.utils import parsedate_to_datetime from typing import Optional -import httpx - -from app.scrapers.truth_social import _process_entry, _post_to_ws_payload -from app.ws.manager import manager +from app.scrapers.truth_social import ( + NOT_MODIFIED, + _known_external_ids, + _process_entry, + dispatch_post, +) logger = logging.getLogger(__name__) @@ -41,17 +43,32 @@ NS = { last_successful_poll_at: Optional[datetime] = None last_poll_error: Optional[str] = None +# Conditional-GET validators (same scheme as truth_social.py โ€” most polls +# come back 304 and skip the download + XML parse entirely). +_etag: Optional[str] = None +_last_modified: Optional[str] = None -async def _fetch_feed() -> Optional[str]: + +async def _fetch_feed(): + """Fetch the RSS body. Returns str, NOT_MODIFIED (304), or None on error.""" + global _etag, _last_modified headers = { "User-Agent": "Mozilla/5.0 (compatible; TrumpSignal/1.0)", "Accept": "application/rss+xml, application/xml", } + if _etag: + headers["If-None-Match"] = _etag + if _last_modified: + headers["If-Modified-Since"] = _last_modified try: - async with httpx.AsyncClient(timeout=20, follow_redirects=True) as client: - resp = await client.get(FEED_URL, headers=headers) - resp.raise_for_status() - return resp.text + from app.services.http_client import get_client + resp = await get_client().get(FEED_URL, headers=headers, timeout=20) + if resp.status_code == 304: + return NOT_MODIFIED + resp.raise_for_status() + _etag = resp.headers.get("etag") + _last_modified = resp.headers.get("last-modified") + return resp.text except Exception as exc: # Include type name โ€” httpx often raises bare ConnectError/RemoteProtocolError # with empty .args, which formats as just "Failed to fetch ..." with no body. @@ -105,6 +122,11 @@ async def poll_trumpstruth(db_session_factory) -> None: global last_successful_poll_at, last_poll_error raw = await _fetch_feed() + if raw is NOT_MODIFIED: + # Feed unchanged โ€” successful cycle, nothing to do. + last_successful_poll_at = datetime.now(timezone.utc) + last_poll_error = None + return if raw is None: last_poll_error = "fetch_feed returned None" return @@ -128,41 +150,26 @@ async def poll_trumpstruth(db_session_factory) -> None: async with db_session_factory() as db: try: - new_posts = [] + known_ids = await _known_external_ids(entries, db) + # Newest-first: commit + dispatch each new post immediately so an + # actionable post never queues behind older entries' AI analysis. + # dispatch_post is the shared fan-out (WS + Telegram + X + trade) + # from truth_social.py โ€” delivery must not depend on which poller + # wins the race. for entry in entries: try: - post = await _process_entry(entry, db) + post = await _process_entry(entry, db, known_ids) if post: - new_posts.append(post) + await db.commit() + logger.info("[trumpstruth] beat CNN โ€” new post id=%d", + post.id) + await dispatch_post(post, db) except Exception as exc: logger.error("trumpstruth: error on entry %s: %s", entry.get("id"), exc) - if new_posts: - await db.commit() - for post in new_posts: - await manager.broadcast(_post_to_ws_payload(post)) - logger.info("[trumpstruth] beat CNN โ€” new post id=%d: %s", - post.id, post.text[:60]) - # Telegram fan-out โ€” matches truth_social.py. Without this, - # whichever poller wins the race determines whether users - # get pushed โ€” flaky 50% delivery. - try: - from app.services.telegram import notify_signal - notify_signal(post) - except Exception as exc: - logger.warning("Telegram notify failed for post %d: %s", post.id, exc) - try: - from app.services.x_poster import notify_x_signal - notify_x_signal(post) - except Exception as exc: - logger.warning("X notify failed for post %d: %s", post.id, exc) - try: - from app.services.bot_engine import process_post - await process_post(post, db) - except Exception as exc: - logger.error("process_post failed for post %d: %s", - post.id, exc) + # Capture any remaining writes (entry-filter stub rows). + await db.commit() last_successful_poll_at = datetime.now(timezone.utc) last_poll_error = None except Exception as exc: diff --git a/app/scrapers/truth_social.py b/app/scrapers/truth_social.py index e40708e..afb1ac9 100644 --- a/app/scrapers/truth_social.py +++ b/app/scrapers/truth_social.py @@ -11,7 +11,6 @@ import re from datetime import datetime, timezone from typing import Optional -import httpx from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -29,6 +28,16 @@ ARCHIVE_URL = "https://ix.cnn.io/data/truth-social/truth_archive.json" last_successful_poll_at: Optional[datetime] = None last_poll_error: Optional[str] = None +# Conditional-GET validators from the last 200 response. The archive is 30k+ +# posts (~MBs of JSON); CNN only regenerates it every ~5 min, so most polls +# can be answered with a 304 and skip download + parse entirely. +_etag: Optional[str] = None +_last_modified: Optional[str] = None + +# Sentinel returned by _fetch_archive on HTTP 304 โ€” distinct from None +# (fetch failure) so the poller can count it as a successful, no-work cycle. +NOT_MODIFIED = object() + def _strip_html(text: str) -> str: text = re.sub(r"<[^>]+>", " ", text) @@ -52,16 +61,29 @@ def _parse_dt(iso: str) -> datetime: return datetime.now(timezone.utc).replace(tzinfo=None) -async def _fetch_archive() -> Optional[list]: +async def _fetch_archive(conditional: bool = True): + """Fetch the archive JSON. Returns a list, NOT_MODIFIED (304), or None + on error. `conditional=False` (backfill) always downloads the full body + so a poller-set ETag can't starve the startup backfill.""" + global _etag, _last_modified headers = { "User-Agent": "Mozilla/5.0 (compatible; TrumpSignal/1.0)", "Accept": "application/json", } + if conditional: + if _etag: + headers["If-None-Match"] = _etag + if _last_modified: + headers["If-Modified-Since"] = _last_modified try: - async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: - resp = await client.get(ARCHIVE_URL, headers=headers) - resp.raise_for_status() - return resp.json() + from app.services.http_client import get_client + resp = await get_client().get(ARCHIVE_URL, headers=headers, timeout=30) + if conditional and resp.status_code == 304: + return NOT_MODIFIED + resp.raise_for_status() + _etag = resp.headers.get("etag") + _last_modified = resp.headers.get("last-modified") + return resp.json() except Exception as exc: # Include type name โ€” httpx often raises bare ConnectError/TimeoutException # with empty .args, which used to log as just "Failed to fetch CNN archive:" @@ -71,9 +93,27 @@ async def _fetch_archive() -> Optional[list]: return None -async def _process_entry(entry: dict, db: AsyncSession) -> Optional[Post]: +async def _known_external_ids(entries: list, db: AsyncSession) -> set: + """One batch query for the dedup pass. On a typical poll all ~50 entries + already exist, so this replaces 50 per-entry SELECTs with 1.""" + ids = [hashlib.md5(str(e["id"]).encode()).hexdigest() for e in entries] + if not ids: + return set() + rows = await db.execute( + select(Post.external_id).where(Post.external_id.in_(ids))) + return {r[0] for r in rows} + + +async def _process_entry(entry: dict, db: AsyncSession, + known_ids: Optional[set] = None) -> Optional[Post]: external_id = hashlib.md5(str(entry["id"]).encode()).hexdigest() + # Fast path: pre-fetched batch dedup set. New (unseen) ids still get the + # confirming SELECT below โ€” protects against a concurrent insert by the + # other poller between the batch query and this entry. + if known_ids is not None and external_id in known_ids: + return None + result = await db.execute(select(Post).where(Post.external_id == external_id)) if result.scalar_one_or_none(): return None @@ -198,10 +238,40 @@ def _post_to_ws_payload(post: Post) -> dict: } +async def dispatch_post(post: Post, db: AsyncSession) -> None: + """Broadcast + fan-out + trade for one freshly committed post. Shared by + both pollers so delivery doesn't depend on which source wins the race.""" + await manager.broadcast(_post_to_ws_payload(post)) + logger.info("Saved new post id=%d: %s", post.id, post.text[:60]) + # Telegram fan-out (fire-and-forget). _dispatch filters internally: + # buy/short โ†’ per-subscriber + public channel; relevant-but-hold โ†’ + # public channel only; noise โ†’ dropped. + try: + from app.services.telegram import notify_signal + notify_signal(post) + except Exception as exc: + logger.warning("Telegram notify failed for post %d: %s", post.id, exc) + try: + from app.services.x_poster import notify_x_signal + notify_x_signal(post) + except Exception as exc: + logger.warning("X notify failed for post %d: %s", post.id, exc) + try: + from app.services.bot_engine import process_post + await process_post(post, db) + except Exception as exc: + logger.error("process_post failed for post %d: %s", post.id, exc) + + async def poll_truth_social(db_session_factory) -> None: global last_successful_poll_at, last_poll_error logger.info("Polling CNN Truth Social archive...") entries = await _fetch_archive() + if entries is NOT_MODIFIED: + # Archive unchanged since last poll โ€” successful cycle, nothing to do. + last_successful_poll_at = datetime.now(timezone.utc) + last_poll_error = None + return if not entries: last_poll_error = "fetch_archive returned empty" return @@ -212,39 +282,24 @@ async def poll_truth_social(db_session_factory) -> None: async with db_session_factory() as db: try: - new_posts = [] + known_ids = await _known_external_ids(recent, db) + found_new = False + # Entries are newest-first. Commit + dispatch each new post + # IMMEDIATELY rather than after the whole batch โ€” an actionable + # post must not wait behind the AI analysis of older entries. for entry in recent: try: - post = await _process_entry(entry, db) + post = await _process_entry(entry, db, known_ids) if post: - new_posts.append(post) + found_new = True + await db.commit() + await dispatch_post(post, db) except Exception as exc: logger.error("Error processing entry %s: %s", entry.get("id"), exc) - if new_posts: - await db.commit() - for post in new_posts: - await manager.broadcast(_post_to_ws_payload(post)) - logger.info("Saved new post id=%d: %s", post.id, post.text[:60]) - # Telegram fan-out (fire-and-forget). _dispatch filters - # internally: buy/short โ†’ per-subscriber + public channel; - # relevant-but-hold โ†’ public channel only; noise โ†’ dropped. - try: - from app.services.telegram import notify_signal - notify_signal(post) - except Exception as exc: - logger.warning("Telegram notify failed for post %d: %s", post.id, exc) - try: - from app.services.x_poster import notify_x_signal - notify_x_signal(post) - except Exception as exc: - logger.warning("X notify failed for post %d: %s", post.id, exc) - try: - from app.services.bot_engine import process_post - await process_post(post, db) - except Exception as exc: - logger.error("process_post failed for post %d: %s", post.id, exc) - else: + # Capture any remaining writes (entry-filter stub rows). + await db.commit() + if not found_new: logger.info("No new posts found.") # Mark a successful poll cycle (separate from "found new posts"). last_successful_poll_at = datetime.now(timezone.utc) @@ -258,7 +313,7 @@ async def poll_truth_social(db_session_factory) -> None: async def backfill_history(db_session_factory, limit: int = 500) -> None: """One-time backfill of historical posts (no Claude analysis, no price impact).""" logger.info("Starting historical backfill (limit=%d)...", limit) - entries = await _fetch_archive() + entries = await _fetch_archive(conditional=False) if not entries: logger.error("Backfill failed: could not fetch archive") return @@ -268,10 +323,10 @@ async def backfill_history(db_session_factory, limit: int = 500) -> None: async with db_session_factory() as db: try: + known_ids = await _known_external_ids(to_process, db) for entry in to_process: external_id = hashlib.md5(str(entry["id"]).encode()).hexdigest() - result = await db.execute(select(Post).where(Post.external_id == external_id)) - if result.scalar_one_or_none(): + if external_id in known_ids: continue text = _strip_html(entry.get("content") or "").strip() diff --git a/app/services/adoption.py b/app/services/adoption.py index 2ab831d..8553110 100644 --- a/app/services/adoption.py +++ b/app/services/adoption.py @@ -244,6 +244,16 @@ async def _adopt_locked(wallet_l: str, asset_u: str, mode_n: str) -> AdoptionRes "no Hyperliquid position to manage. Turn off paper mode in " "Settings to use /adopt.") + # Macro Vibes (System-2) must be ENABLED to hand a position to the bot. + # /adopt is the entry point to sys2 management, so honouring the + # macro_enabled switch here is what makes that toggle real โ€” otherwise + # a user who turned Macro Vibes OFF could still /adopt and the bot would + # manage it, contradicting their setting. + if not getattr(sub, "macro_enabled", False): + raise AdoptionError("macro_disabled", + "Macro Vibes is turned off for this wallet. Enable Macro Vibes " + "in Settings before using /adopt.") + # System-2 circuit breaker. Same gate the auto-open path used to run: # if recent losses tripped the sys2 breaker, block new adoptions for # the lockout window. Otherwise the breaker would be useless under diff --git a/app/services/binance.py b/app/services/binance.py index 2dc2bf2..2a3f61a 100644 --- a/app/services/binance.py +++ b/app/services/binance.py @@ -4,7 +4,6 @@ import logging from datetime import datetime, timezone from typing import Optional -import httpx import websockets from app.config import settings @@ -127,19 +126,19 @@ async def fetch_historical(asset: str, symbol: str, interval: str = "1m", limit: """Fetch historical klines from Binance REST API to pre-fill price_store.""" url = f"{settings.binance_rest_url}/api/v3/klines?symbol={symbol.upper()}&interval={interval}&limit={limit}" try: - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.get(url) - resp.raise_for_status() - for row in resp.json(): - candle = { - "time": row[0], # open time ms - "open": float(row[1]), - "high": float(row[2]), - "low": float(row[3]), - "close": float(row[4]), - "volume": float(row[5]), - } - price_store.update(asset, candle) + from app.services.http_client import get_client + resp = await get_client().get(url, timeout=15) + resp.raise_for_status() + for row in resp.json(): + candle = { + "time": row[0], # open time ms + "open": float(row[1]), + "high": float(row[2]), + "low": float(row[3]), + "close": float(row[4]), + "volume": float(row[5]), + } + price_store.update(asset, candle) logger.info("Loaded %d historical %s candles for %s", limit, interval, asset) except Exception as exc: logger.error("Failed to fetch historical data for %s: %s", asset, exc) diff --git a/app/services/crypto.py b/app/services/crypto.py index fb3541e..c69074e 100644 --- a/app/services/crypto.py +++ b/app/services/crypto.py @@ -1,62 +1,141 @@ """ Envelope encryption for HL API private keys. -Plaintext keys never touch disk: stored values are Fernet-encrypted with a KEK -loaded from env (ENCRYPTION_KEY). Rotate KEK โ†’ re-encrypt all keys offline. +Plaintext keys never touch disk: stored values are Fernet-encrypted with a key +derived from the env KEK (ENCRYPTION_KEY). + +Blob formats: + enc:v2:: โ€” current. Per-blob random 16-byte + salt; Fernet key = PBKDF2-HMAC-SHA256(KEK, salt, 600k iters). Derived + keys are cached per salt so steady-state decryption pays the KDF once. + enc:v1: โ€” legacy. Fernet key = single unsalted + SHA-256(KEK). Read-only: decrypt still works, encrypt always writes v2. + Upgrade rows with scripts/reencrypt_keys.py (H4 fix). + โ€” pre-encryption rows. Refused in prod. + +Rotate KEK โ†’ re-encrypt all keys offline (scripts/reencrypt_keys.py). """ import base64 import hashlib import logging -from typing import Optional +import os +from typing import Dict, Optional from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC from app.config import settings logger = logging.getLogger(__name__) +# OWASP-recommended floor for PBKDF2-HMAC-SHA256. The KEK is already a +# high-entropy secret, so the KDF mainly buys defence-in-depth against a +# weak / partially-leaked / reused KEK. ~0.2-0.4s, paid once per unique salt. +_PBKDF2_ITERATIONS = 600_000 +_SALT_BYTES = 16 -def _derive_fernet_key(raw: str) -> bytes: - """Accept any reasonably-long secret and derive a valid 32-byte Fernet key.""" +ENC_PREFIX_V1 = "enc:v1:" +ENC_PREFIX_V2 = "enc:v2:" +# Kept for older imports / scripts that reference the original name. +ENC_PREFIX = ENC_PREFIX_V1 + + +def _check_kek(raw: str) -> str: if not raw or len(raw) < 32: raise RuntimeError( "ENCRYPTION_KEY must be set to at least 32 random chars (e.g. `openssl rand -hex 32`)" ) - digest = hashlib.sha256(raw.encode("utf-8")).digest() + return raw + + +def _derive_v1_key(raw: str) -> bytes: + """Legacy: single unsalted SHA-256 of the KEK.""" + digest = hashlib.sha256(_check_kek(raw).encode("utf-8")).digest() return base64.urlsafe_b64encode(digest) -_fernet: Optional[Fernet] = None +def _derive_v2_key(raw: str, salt: bytes) -> bytes: + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + iterations=_PBKDF2_ITERATIONS, + ) + return base64.urlsafe_b64encode(kdf.derive(_check_kek(raw).encode("utf-8"))) -def _cipher() -> Fernet: - global _fernet - if _fernet is None: - _fernet = Fernet(_derive_fernet_key(settings.encryption_key)) - return _fernet +_fernet_v1: Optional[Fernet] = None +# salt โ†’ Fernet. One entry per unique salt actually seen: encryption reuses a +# single process-lifetime salt, decryption adds one per distinct stored blob. +_fernet_v2_cache: Dict[bytes, Fernet] = {} +_V2_CACHE_MAX = 4096 + +# Salt for NEW encryptions in this process โ€” generated once so the encrypt +# path pays the 600k-iteration KDF a single time per process, while blobs +# written by other processes/runs still carry their own salt. +_encrypt_salt: Optional[bytes] = None -# Prefix lets us distinguish encrypted blobs from any legacy plaintext rows during migration -ENC_PREFIX = "enc:v1:" +def _cipher_v1() -> Fernet: + global _fernet_v1 + if _fernet_v1 is None: + _fernet_v1 = Fernet(_derive_v1_key(settings.encryption_key)) + return _fernet_v1 + + +def _cipher_v2(salt: bytes) -> Fernet: + cached = _fernet_v2_cache.get(salt) + if cached is not None: + return cached + f = Fernet(_derive_v2_key(settings.encryption_key, salt)) + if len(_fernet_v2_cache) >= _V2_CACHE_MAX: + _fernet_v2_cache.clear() + _fernet_v2_cache[salt] = f + return f def encrypt_api_key(plaintext: str) -> str: - token = _cipher().encrypt(plaintext.encode("utf-8")).decode("utf-8") - return ENC_PREFIX + token + global _encrypt_salt + if _encrypt_salt is None: + _encrypt_salt = os.urandom(_SALT_BYTES) + salt_b64 = base64.urlsafe_b64encode(_encrypt_salt).decode("ascii") + token = _cipher_v2(_encrypt_salt).encrypt(plaintext.encode("utf-8")).decode("utf-8") + return f"{ENC_PREFIX_V2}{salt_b64}:{token}" def decrypt_api_key(stored: str) -> str: if not stored: raise ValueError("Empty api key") - if not stored.startswith(ENC_PREFIX): - # Legacy plaintext row (from before encryption was added). Refuse to use in prod. - if settings.environment == "production": - raise RuntimeError( - "Found legacy-plaintext HL key; run migration script before production" - ) - logger.warning("Reading LEGACY plaintext HL key โ€” migrate ASAP") - return stored - try: - return _cipher().decrypt(stored[len(ENC_PREFIX):].encode("utf-8")).decode("utf-8") - except InvalidToken as exc: - raise RuntimeError("HL key decryption failed โ€” wrong ENCRYPTION_KEY?") from exc + + if stored.startswith(ENC_PREFIX_V2): + rest = stored[len(ENC_PREFIX_V2):] + try: + salt_b64, token = rest.split(":", 1) + salt = base64.urlsafe_b64decode(salt_b64.encode("ascii")) + except Exception as exc: + raise RuntimeError("Malformed enc:v2 blob") from exc + try: + return _cipher_v2(salt).decrypt(token.encode("utf-8")).decode("utf-8") + except InvalidToken as exc: + raise RuntimeError("HL key decryption failed โ€” wrong ENCRYPTION_KEY?") from exc + + if stored.startswith(ENC_PREFIX_V1): + try: + return _cipher_v1().decrypt( + stored[len(ENC_PREFIX_V1):].encode("utf-8")).decode("utf-8") + except InvalidToken as exc: + raise RuntimeError("HL key decryption failed โ€” wrong ENCRYPTION_KEY?") from exc + + # Legacy plaintext row (from before encryption was added). Refuse to use in prod. + if settings.environment == "production": + raise RuntimeError( + "Found legacy-plaintext HL key; run migration script before production" + ) + logger.warning("Reading LEGACY plaintext HL key โ€” migrate ASAP") + return stored + + +def is_current_format(stored: Optional[str]) -> bool: + """True if the blob is already enc:v2 (used by scripts/reencrypt_keys.py).""" + return bool(stored) and stored.startswith(ENC_PREFIX_V2) diff --git a/app/services/hl_price_feed.py b/app/services/hl_price_feed.py index 4f26776..d942ebe 100644 --- a/app/services/hl_price_feed.py +++ b/app/services/hl_price_feed.py @@ -31,7 +31,6 @@ import time from datetime import datetime, timezone from typing import Optional -import httpx from app.services.price_store import price_store from app.ws.manager import manager @@ -96,10 +95,10 @@ async def _tick() -> None: """Single price fetch + dispatch cycle for all HL_PRICE_ASSETS.""" now_ms = int(time.time() * 1000) - async with httpx.AsyncClient(timeout=4.0) as c: - r = await c.post(HL_API_URL, json={"type": "allMids"}) - r.raise_for_status() - mids: dict = r.json() # {"BTC": "74541.0", "HYPE": "13.5", โ€ฆ} + from app.services.http_client import get_client + r = await get_client().post(HL_API_URL, json={"type": "allMids"}, timeout=4.0) + r.raise_for_status() + mids: dict = r.json() # {"BTC": "74541.0", "HYPE": "13.5", โ€ฆ} # Feed is alive the moment we successfully fetch mids, even if a specific # asset is momentarily absent from the response. diff --git a/app/services/http_client.py b/app/services/http_client.py new file mode 100644 index 0000000..ae5a09c --- /dev/null +++ b/app/services/http_client.py @@ -0,0 +1,37 @@ +"""Shared pooled httpx.AsyncClient. + +Hot paths (scrapers, Telegram send/poll, price feeds, X poster) used to build +a fresh AsyncClient per request, paying a TCP+TLS handshake every time. This +module owns one process-wide client with keep-alive pooling; callers override +the timeout per request (`client.get(url, timeout=10)`). + +Lifecycle: lazily created on first use; main.py's lifespan closes it on +shutdown. Low-frequency daily jobs (KOL/macro fetchers) may keep their own +ad-hoc clients โ€” pooling only matters on the per-second paths. +""" +import logging +from typing import Optional + +import httpx + +logger = logging.getLogger(__name__) + +_client: Optional[httpx.AsyncClient] = None + + +def get_client() -> httpx.AsyncClient: + global _client + if _client is None or _client.is_closed: + _client = httpx.AsyncClient( + timeout=httpx.Timeout(20.0), + follow_redirects=True, + limits=httpx.Limits(max_connections=50, max_keepalive_connections=20), + ) + return _client + + +async def aclose() -> None: + global _client + if _client is not None and not _client.is_closed: + await _client.aclose() + _client = None diff --git a/app/services/kol_analysis.py b/app/services/kol_analysis.py index 7fe6a29..f874d19 100644 --- a/app/services/kol_analysis.py +++ b/app/services/kol_analysis.py @@ -315,6 +315,8 @@ async def extract_kol_signal( if post_type not in valid_post_types: post_type = "other" + tier = _derive_tier(cleaned, tvt_score) + return { "summary": (data.get("summary") or "").strip() or None, "post_type": post_type, @@ -322,6 +324,33 @@ async def extract_kol_signal( "talks_vs_trades_score": tvt_score, # Keep old boolean for any callers that still check it "talks_vs_trades_flag": tvt_score >= 0.5, + # tier mirrors x_analysis' vocabulary (trade_signal / directional / + # noise) so blog/substack/podcast posts get the same SIGNAL/VIEW UI + # badges + the "Signals only" filter that Twitter posts already have. + "tier": tier, "model": model, "version": ANALYSIS_VERSION, } + + +def _derive_tier(tickers: list[dict], tvt_score: float) -> str: + """Map kol_analysis output โ†’ the trade_signal/directional/noise tiers that + x_analysis emits directly. Non-Twitter analyzers don't ask the model for a + tier, so we derive one from the per-ticker conviction + the talks-vs-trades + (divergence) score. Without this, blog/substack/podcast rows have tier=NULL + and the "Signals only" filter + SIGNAL/VIEW badges never apply to them. + + * directional ticker = an explicit non-"mention" action (buy/sell/ + reduce/bullish/bearish). + * trade_signal = high conviction (>= 0.6) on a directional ticker, or a + strong talks-vs-trades divergence (>= 0.6) โ€” the platform's top signal. + * directional = a directional view exists but below the trade_signal bar. + * noise = no directional ticker and no notable divergence. + """ + directional = [t for t in tickers if (t.get("action") or "mention") != "mention"] + max_conv = max((float(t.get("conviction") or 0) for t in directional), default=0.0) + if max_conv >= 0.6 or tvt_score >= 0.6: + return "trade_signal" + if directional or tvt_score >= 0.5: + return "directional" + return "noise" diff --git a/app/services/kol_substack.py b/app/services/kol_substack.py index c80a4b9..c09aee0 100644 --- a/app/services/kol_substack.py +++ b/app/services/kol_substack.py @@ -468,6 +468,9 @@ async def _ingest_kol( # Extended analysis fields (migration 027) row.post_type = result.get("post_type") row.talks_vs_trades_flag = bool(result.get("talks_vs_trades_flag", False)) + # tier (trade_signal/directional/noise) so the "Signals only" + # filter + SIGNAL/VIEW badges work for non-Twitter KOLs too. + row.tier = result.get("tier") stats["analyzed"] += 1 except Exception as e: logger.warning("[kol_substack] analysis failed for %s post %s: %s", diff --git a/app/services/signed_request.py b/app/services/signed_request.py index 0d4c62b..91b2bdf 100644 --- a/app/services/signed_request.py +++ b/app/services/signed_request.py @@ -144,3 +144,58 @@ def verify_signed_request_any( last_exc = exc if last_exc is not None: raise last_exc + + +# โ”€โ”€ Signed-read credential extraction (C3) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Read endpoints historically took the signature as `?ts=&sig=` query params, +# which leaks signatures into access logs / proxies / browser history. The +# canonical transport is now the X-Sig-Ts / X-Sig-Sig HEADERS; the query +# params are kept as a deprecated fallback so older clients keep working. +from dataclasses import dataclass + +from fastapi import Header, Query + + +@dataclass +class SignedReadCreds: + ts: int + sig: str + + +def signed_read_creds( + ts: Optional[int] = Query(default=None, deprecated=True, + description="DEPRECATED โ€” use X-Sig-Ts header"), + sig: Optional[str] = Query(default=None, deprecated=True, + description="DEPRECATED โ€” use X-Sig-Sig header"), + x_sig_ts: Optional[int] = Header(default=None, alias="X-Sig-Ts", + description="Signed timestamp (ms)"), + x_sig_sig: Optional[str] = Header(default=None, alias="X-Sig-Sig", + description="EIP-191 signature"), +) -> SignedReadCreds: + """FastAPI dependency: required signed-read credentials. + + Headers win over query params when both are present.""" + t = x_sig_ts if x_sig_ts is not None else ts + s = x_sig_sig if x_sig_sig else sig + if t is None or not s: + raise HTTPException( + 422, "Missing signed-read credentials (X-Sig-Ts / X-Sig-Sig headers)") + return SignedReadCreds(ts=t, sig=s) + + +def optional_signed_read_creds( + # Legacy query names used by /telegram/{wallet}/status. + timestamp: Optional[int] = Query(default=None, deprecated=True, + description="DEPRECATED โ€” use X-Sig-Ts header"), + signature: Optional[str] = Query(default=None, deprecated=True, + description="DEPRECATED โ€” use X-Sig-Sig header"), + x_sig_ts: Optional[int] = Header(default=None, alias="X-Sig-Ts"), + x_sig_sig: Optional[str] = Header(default=None, alias="X-Sig-Sig"), +) -> Optional[SignedReadCreds]: + """Like signed_read_creds but returns None when absent โ€” for endpoints + that serve a redacted response to unauthenticated callers.""" + t = x_sig_ts if x_sig_ts is not None else timestamp + s = x_sig_sig if x_sig_sig else signature + if t is None or not s: + return None + return SignedReadCreds(ts=t, sig=s) diff --git a/app/services/telegram.py b/app/services/telegram.py index a7d7351..73f4c9c 100644 --- a/app/services/telegram.py +++ b/app/services/telegram.py @@ -27,7 +27,6 @@ import logging from datetime import datetime, timezone from typing import Optional -import httpx from sqlalchemy import select, update from app.config import settings @@ -274,8 +273,8 @@ async def send_message(chat_id: int | str, text: str, *, if reply_markup is not None: payload["reply_markup"] = reply_markup try: - async with httpx.AsyncClient(timeout=10) as client: - r = await client.post(url, json=payload) + from app.services.http_client import get_client + r = await get_client().post(url, json=payload, timeout=10) if r.status_code != 200: logger.warning("Telegram sendMessage failed chat=%s status=%d body=%s", chat_id, r.status_code, r.text[:200]) @@ -302,8 +301,8 @@ async def edit_message(chat_id: int, message_id: int, text: str, *, if reply_markup is not None: payload["reply_markup"] = reply_markup try: - async with httpx.AsyncClient(timeout=10) as client: - r = await client.post(url, json=payload) + from app.services.http_client import get_client + r = await get_client().post(url, json=payload, timeout=10) if r.status_code != 200: # Telegram returns 400 on "message is not modified" โ€” harmless. if "message is not modified" not in r.text: @@ -331,8 +330,8 @@ async def answer_callback(callback_query_id: str, text: str = "", payload["text"] = text payload["show_alert"] = show_alert try: - async with httpx.AsyncClient(timeout=10) as client: - await client.post(url, json=payload) + from app.services.http_client import get_client + await get_client().post(url, json=payload, timeout=10) return True except Exception as exc: logger.debug("Telegram answerCallback exception: %s", exc) diff --git a/app/services/telegram_bot.py b/app/services/telegram_bot.py index c68303c..dccaf1f 100644 --- a/app/services/telegram_bot.py +++ b/app/services/telegram_bot.py @@ -1007,15 +1007,14 @@ async def run_bot_loop() -> None: # them without processing, then start the real loop fresh. try: drain_url = TG_API.format(token=token, method="getUpdates") - async with httpx.AsyncClient(timeout=10) as client: - r = await client.get(drain_url, params={"timeout": 0, "limit": 100}) + from app.services.http_client import get_client + r = await get_client().get(drain_url, params={"timeout": 0, "limit": 100}, timeout=10) if r.status_code == 200: pending = r.json().get("result", []) if pending: drain_offset = pending[-1]["update_id"] + 1 # ACK by sending offset back โ€” Telegram won't re-deliver these. - async with httpx.AsyncClient(timeout=10) as client: - await client.get(drain_url, params={"timeout": 0, "offset": drain_offset}) + await get_client().get(drain_url, params={"timeout": 0, "offset": drain_offset}, timeout=10) logger.info( "Startup drain: skipped %d stale update(s), offset now %d", len(pending), drain_offset, @@ -1032,8 +1031,8 @@ async def run_bot_loop() -> None: params: dict = {"timeout": 25} if offset is not None: params["offset"] = offset - async with httpx.AsyncClient(timeout=35) as client: - r = await client.get(url, params=params) + from app.services.http_client import get_client + r = await get_client().get(url, params=params, timeout=35) if r.status_code != 200: logger.warning("Telegram getUpdates HTTP %d: %s", r.status_code, r.text[:200]) await asyncio.sleep(backoff) diff --git a/app/services/x_poster.py b/app/services/x_poster.py index 2a97abb..c1752a6 100644 --- a/app/services/x_poster.py +++ b/app/services/x_poster.py @@ -33,7 +33,6 @@ import urllib.parse from datetime import datetime, timezone from typing import Optional -import httpx from sqlalchemy import select from app.config import settings @@ -146,8 +145,8 @@ async def _post_tweet(text: str, reply_to: Optional[str] = None) -> Optional[str "Content-Type": "application/json", } try: - async with httpx.AsyncClient(timeout=10) as client: - resp = await client.post(TWEET_URL, json=payload, headers=headers) + from app.services.http_client import get_client + resp = await get_client().post(TWEET_URL, json=payload, headers=headers, timeout=10) if resp.status_code in (200, 201): _record_sent() tid = resp.json().get("data", {}).get("id") diff --git a/scripts/reencrypt_keys.py b/scripts/reencrypt_keys.py new file mode 100644 index 0000000..a6d5a01 --- /dev/null +++ b/scripts/reencrypt_keys.py @@ -0,0 +1,90 @@ +""" +Re-encrypt all stored HL API keys to the current enc:v2 format (H4 fix). + +Upgrades, in place: + * enc:v1 blobs (legacy unsalted-SHA256 KEK derivation) + * plaintext rows (pre-encryption era) +to enc:v2 (PBKDF2-HMAC-SHA256, per-blob salt). Rows already in enc:v2 are +skipped โ€” the script is idempotent and safe to re-run. + +KEK rotation: set OLD_ENCRYPTION_KEY in the environment to decrypt with the +old KEK while encrypting with the current ENCRYPTION_KEY. + +Usage (BACK UP THE DB FIRST): + DATABASE_URL=<prod-url> python scripts/reencrypt_keys.py [--dry-run] +""" +import argparse +import asyncio +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sqlalchemy import select + +from app.database import AsyncSessionLocal +from app.models import Subscription +from app.services import crypto + + +async def main(dry_run: bool) -> None: + old_kek = os.environ.get("OLD_ENCRYPTION_KEY") + if old_kek: + # Decrypt with the old KEK, encrypt with the new one. Swapping the + # setting around each call keeps crypto.py's caching simple. + print("KEK rotation mode: decrypting with OLD_ENCRYPTION_KEY") + + upgraded = skipped = failed = 0 + async with AsyncSessionLocal() as db: + rows = (await db.execute( + select(Subscription).where(Subscription.hl_api_key.is_not(None)) + )).scalars().all() + print(f"{len(rows)} subscription(s) with a stored HL key") + + for sub in rows: + stored = sub.hl_api_key + if crypto.is_current_format(stored) and not old_kek: + skipped += 1 + continue + try: + if old_kek: + current = crypto.settings.encryption_key + crypto.settings.encryption_key = old_kek + crypto._fernet_v1 = None + crypto._fernet_v2_cache.clear() + try: + plaintext = crypto.decrypt_api_key(stored) + finally: + crypto.settings.encryption_key = current + crypto._fernet_v1 = None + crypto._fernet_v2_cache.clear() + crypto._encrypt_salt = None + else: + plaintext = crypto.decrypt_api_key(stored) + new_blob = crypto.encrypt_api_key(plaintext) + # Round-trip check before touching the row. + assert crypto.decrypt_api_key(new_blob) == plaintext + if not dry_run: + sub.hl_api_key = new_blob + upgraded += 1 + print(f" {sub.wallet_address}: upgraded" + f"{' (dry-run)' if dry_run else ''}") + except Exception as exc: + failed += 1 + print(f" {sub.wallet_address}: FAILED โ€” {type(exc).__name__}: {exc}") + + if not dry_run and upgraded: + await db.commit() + print("Committed.") + + print(f"Done: upgraded={upgraded} skipped(already v2)={skipped} failed={failed}") + if failed: + sys.exit(1) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true", + help="report what would change without writing") + args = ap.parse_args() + asyncio.run(main(args.dry_run)) diff --git a/tests/test_adoption.py b/tests/test_adoption.py index 2fafc25..e7215cc 100644 --- a/tests/test_adoption.py +++ b/tests/test_adoption.py @@ -155,7 +155,7 @@ def test_adoption_error_codes_used_by_callers_are_all_defined(): # These are the codes referenced from app/api/positions.py + # app/services/telegram_bot.py. Update both sites if you add one. promised_codes = { - "no_subscription", "no_hl_key", "paper_mode", + "no_subscription", "no_hl_key", "paper_mode", "macro_disabled", "key_decrypt_failed", "hl_read_failed", "bad_mode", "circuit_breaker", "already_adopted", "concurrency_cap", diff --git a/tests/test_crypto.py b/tests/test_crypto.py new file mode 100644 index 0000000..153bcca --- /dev/null +++ b/tests/test_crypto.py @@ -0,0 +1,58 @@ +"""enc:v2 envelope encryption (H4): roundtrip, legacy v1 compat, format checks.""" +import base64 +import hashlib + +import pytest +from cryptography.fernet import Fernet + +from app.services import crypto + + +@pytest.fixture(autouse=True) +def _kek(monkeypatch): + monkeypatch.setattr(crypto.settings, "encryption_key", "k" * 64) + # Reset module caches so each test derives from the patched KEK. + crypto._fernet_v1 = None + crypto._fernet_v2_cache.clear() + crypto._encrypt_salt = None + yield + crypto._fernet_v1 = None + crypto._fernet_v2_cache.clear() + crypto._encrypt_salt = None + + +def test_v2_roundtrip(): + blob = crypto.encrypt_api_key("0x" + "ab" * 32) + assert blob.startswith(crypto.ENC_PREFIX_V2) + assert crypto.decrypt_api_key(blob) == "0x" + "ab" * 32 + assert crypto.is_current_format(blob) + # Fits the hl_api_key String(256) column. + assert len(blob) <= 256 + + +def test_v1_blob_still_decrypts(): + # Build a v1 blob exactly the way the legacy code did. + digest = hashlib.sha256(("k" * 64).encode()).digest() + f = Fernet(base64.urlsafe_b64encode(digest)) + blob = crypto.ENC_PREFIX_V1 + f.encrypt(b"secret-key").decode() + assert not crypto.is_current_format(blob) + assert crypto.decrypt_api_key(blob) == "secret-key" + + +def test_wrong_kek_raises(monkeypatch): + blob = crypto.encrypt_api_key("topsecret") + monkeypatch.setattr(crypto.settings, "encryption_key", "x" * 64) + crypto._fernet_v2_cache.clear() + with pytest.raises(RuntimeError): + crypto.decrypt_api_key(blob) + + +def test_malformed_v2_raises(): + with pytest.raises(RuntimeError): + crypto.decrypt_api_key(crypto.ENC_PREFIX_V2 + "no-salt-separator") + + +def test_plaintext_refused_in_production(monkeypatch): + monkeypatch.setattr(crypto.settings, "environment", "production") + with pytest.raises(RuntimeError): + crypto.decrypt_api_key("raw-plaintext-key") diff --git a/tests/test_kol_tier.py b/tests/test_kol_tier.py new file mode 100644 index 0000000..7139d28 --- /dev/null +++ b/tests/test_kol_tier.py @@ -0,0 +1,38 @@ +"""Unit tests for kol_analysis._derive_tier โ€” the non-Twitter tier derivation +that lets blog/substack/podcast posts get the same trade_signal/directional/ +noise tiers (and SIGNAL/VIEW badges + "Signals only" filter) as Twitter posts. +""" + +from app.services.kol_analysis import _derive_tier + + +def _tk(action="mention", conviction=0.0): + return {"action": action, "conviction": conviction} + + +def test_no_tickers_is_noise(): + assert _derive_tier([], 0.0) == "noise" + + +def test_only_mentions_is_noise(): + assert _derive_tier([_tk("mention", 0.9)], 0.1) == "noise" + + +def test_high_conviction_directional_is_trade_signal(): + assert _derive_tier([_tk("buy", 0.7)], 0.0) == "trade_signal" + + +def test_strong_divergence_is_trade_signal_even_without_ticker(): + assert _derive_tier([], 0.8) == "trade_signal" + + +def test_low_conviction_directional_is_directional(): + assert _derive_tier([_tk("bullish", 0.4)], 0.0) == "directional" + + +def test_moderate_divergence_is_directional(): + assert _derive_tier([_tk("mention", 0.0)], 0.55) == "directional" + + +def test_boundary_conviction_0_6_is_trade_signal(): + assert _derive_tier([_tk("sell", 0.6)], 0.0) == "trade_signal" diff --git a/tests/test_production_readiness.py b/tests/test_production_readiness.py index 1a07281..5673590 100644 --- a/tests/test_production_readiness.py +++ b/tests/test_production_readiness.py @@ -97,10 +97,10 @@ async def test_open_positions_requires_signed_wallet_read(monkeypatch): monkeypatch.setattr(positions, "verify_signed_request_any", fake_verify) db = _Db([[]]) + from app.services.signed_request import SignedReadCreds result = await positions.get_open_positions( wallet="0xABC", - ts=123, - sig="0xsig", + creds=SignedReadCreds(ts=123, sig="0xsig"), db=db, )