""" Generic signal ingestion endpoint. The BTC bottom scanner POSTs a signal here and it flows through the shared execution pipeline: sizing → risk caps → Hyperliquid execution → trailing stop monitor. Unknown sources are accepted into the posts table for audit, but the execution layer fails closed and will not trade them. Why one endpoint instead of many? The trusted scanners and the Trump scraper share the same trade execution path. The downstream code sees a `Post` row with `signal=buy|short`, then checks whether that post source is allowed to trade. Auth: shared secret in X-Ingest-Key header. Fail-closed if INGEST_API_KEY is empty in env (so a fresh deploy can't accidentally accept random POSTs). Source field convention: - "truth" → Trump posts (existing scraper). Reserved — rejected here. - "btc_bottom_reversal" → Bitcoin Bottom scanner - "" → Audit/storage only unless whitelisted in signal_categories.py. """ from __future__ import annotations import hashlib import logging from datetime import datetime, timezone from typing import Optional from fastapi import APIRouter, Depends, Header, HTTPException from pydantic import BaseModel, Field, field_validator from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.config import settings from app.database import get_db from app.models import Post router = APIRouter() logger = logging.getLogger(__name__) # ─── Schema ───────────────────────────────────────────────────────────────── class SignalIngestRequest(BaseModel): """Payload for POST /api/signals/ingest. Mirrors the Post schema closely so a signal here becomes a Post row 1:1, with no field translation. Downstream code is unaware of the source. """ source: str = Field(..., min_length=2, max_length=32, description="Signal source tag, e.g. 'btc_bottom_reversal'. 'truth' is reserved.") external_id: str = Field(..., min_length=4, max_length=64, description="Caller-supplied unique key. Used for idempotent retries.") text: str = Field(..., min_length=1, max_length=4000, description="Human-readable description of why the signal fired. Shown in UI.") signal: str = Field(..., description="'buy' or 'short' — non-actionable signals should not be ingested.") target_asset: str = Field(..., min_length=2, max_length=16, description="Hyperliquid perp ticker, e.g. 'SOL', 'BTC'.") confidence: int = Field(..., ge=0, le=100, description="0–100. Drives position sizing (≥90 boosts multiplier).") category: str = Field(..., min_length=2, max_length=24, description="Subtype within source, e.g. 'btc_bottom_reversal_long'.") expected_move_pct: Optional[float] = Field(None, ge=0, le=100, description="Optional: caller's expected % move. UI display only.") invalidation_price: Optional[float] = Field(None, ge=0, description="Optional thesis invalidation level for System 2.") published_at: Optional[datetime] = Field(None, description="Defaults to server now (UTC). Set explicitly only for backfill.") @field_validator("signal") @classmethod def _signal_must_be_directional(cls, v: str) -> str: if v not in ("buy", "short"): raise ValueError(f"signal must be 'buy' or 'short' (got {v!r})") return v @field_validator("source") @classmethod def _no_reserved_sources(cls, v: str) -> str: if v.lower() == "truth": raise ValueError("'truth' is reserved for the Trump scraper. Use a different source tag.") return v.lower() class SignalIngestResponse(BaseModel): status: str # "accepted" | "duplicate" | "skipped" post_id: Optional[int] = None dedup_against: Optional[int] = None note: Optional[str] = None # ─── Auth helper ──────────────────────────────────────────────────────────── def _verify_ingest_key(x_ingest_key: Optional[str]) -> None: """Fail-closed auth check. - INGEST_API_KEY unset → return 503 (endpoint disabled by config) - Header missing or wrong → 401 """ expected = settings.ingest_api_key if not expected: raise HTTPException(503, "signal ingest endpoint is disabled (INGEST_API_KEY not configured)") if not x_ingest_key: raise HTTPException(401, "missing X-Ingest-Key header") if x_ingest_key != expected: raise HTTPException(401, "invalid X-Ingest-Key") # ─── Endpoint ─────────────────────────────────────────────────────────────── @router.post("/signals/ingest", response_model=SignalIngestResponse) async def ingest_signal( body: SignalIngestRequest, x_ingest_key: Optional[str] = Header(None, alias="X-Ingest-Key"), db: AsyncSession = Depends(get_db), ) -> SignalIngestResponse: """Accept a signal from an external trading module. Pipeline behaviour: 1. Auth (shared secret in header) 2. Dedup by external_id (idempotent — same id returns the existing post_id) 3. Insert as a Post row (source = caller-supplied) 4. If the source is whitelisted for trading, hand off to bot_engine.process_post. Otherwise store for audit and return status="skipped". Note: the entry filter ([A] in the pipeline) is SKIPPED for non-truth sources. That filter is tuned for Trump's writing patterns; technical signals are their own catalyst and don't need text-based gating. """ _verify_ingest_key(x_ingest_key) # Hash external_id the same way truth_social.py does — keeps the dedup # key short and uniform across sources. ext_id_hashed = hashlib.md5(f"{body.source}:{body.external_id}".encode()).hexdigest() # Idempotency check existing = await db.execute(select(Post).where(Post.external_id == ext_id_hashed)) prior = existing.scalar_one_or_none() if prior: return SignalIngestResponse( status="duplicate", post_id=prior.id, dedup_against=prior.id, note=f"external_id {body.external_id!r} already ingested", ) # Publication time defaults to server now (naive UTC, matching other rows) pub_naive = (body.published_at or datetime.now(timezone.utc)) if pub_naive.tzinfo is not None: pub_naive = pub_naive.astimezone(timezone.utc).replace(tzinfo=None) post = Post( external_id=ext_id_hashed, text=body.text, source=body.source, published_at=pub_naive, sentiment="bullish" if body.signal == "buy" else "bearish", ai_confidence=body.confidence, relevant=True, signal=body.signal, target_asset=body.target_asset.upper(), category=body.category, expected_move_pct=body.expected_move_pct, invalidation_price=body.invalidation_price, # `analysis_version` doubles as a provenance tag — easy to query in DB analysis_version=f"ingest:{body.source}", prefilter_reason="external_signal", # bypasses entry-filter audit ) db.add(post) await db.commit() await db.refresh(post) logger.info( "Ingested signal: source=%s id=%s → post_id=%d, %s/%s conf=%d category=%s", body.source, body.external_id, post.id, body.signal, body.target_asset, body.confidence, body.category, ) from app.services.signal_categories import is_supported_trading_source if not is_supported_trading_source(post.source): logger.info("Signal %d stored but skipped: unsupported trading source=%s", post.id, post.source) return SignalIngestResponse( status="skipped", post_id=post.id, note=f"source {post.source!r} is not enabled for live trading", ) # Hand off to the same trade pipeline used by Trump posts. # Runs sizing → risk caps → HL trade → trailing-stop monitor. try: from app.services.bot_engine import process_post await process_post(post, db) except Exception as exc: # Don't fail the HTTP response — the post is already in DB, and # process_post failures shouldn't make the external module retry # (which would duplicate signals). Log and return success. logger.error("process_post failed for ingested signal %d: %s", post.id, exc) # Broadcast to UI so the new signal shows up live, same as Trump posts. try: from app.scrapers.truth_social import _post_to_ws_payload from app.ws.manager import manager await manager.broadcast(_post_to_ws_payload(post)) except Exception as exc: logger.warning("WS broadcast failed for signal %d: %s", post.id, exc) # Fan out to Telegram subscribers (fire-and-forget; never blocks ingest). try: from app.services.telegram import notify_signal notify_signal(post) except Exception as exc: logger.warning("Telegram notify failed for signal %d: %s", post.id, exc) # Auto-post a viral "live prediction" tweet (fire-and-forget; no-op unless # X is configured + enabled). Internally gated to source=truth buy/short. try: from app.services.x_poster import notify_x_signal notify_x_signal(post) except Exception as exc: logger.warning("X notify failed for signal %d: %s", post.id, exc) return SignalIngestResponse(status="accepted", post_id=post.id) @router.get("/signals/sources") async def list_sources(db: AsyncSession = Depends(get_db)) -> dict: """List distinct signal sources we've seen, with counts. Helpful for spot-checking that ingestion is working and for the UI to show a source filter without hard-coding the list. """ from sqlalchemy import func rows = await db.execute( select(Post.source, func.count(Post.id), func.max(Post.published_at)) .group_by(Post.source) .order_by(func.count(Post.id).desc()) ) return { "sources": [ {"source": r[0], "count": r[1], "latest": r[2].isoformat() if r[2] else None} for r in rows.all() ] }