k
54884f3e24
Feed-health pass over KOL_FEEDS: - raoulpal: stale Substack (last 2024-05) → Real Vision podcast feed - dampedspring: paywalled (0 entries) → free "Damped Spring 101" Substack - unchained: Cloudflare 403 → canonical Megaphone podcast feed - lynalden: Cloudflare 202 → FeedBurner mirror - glassnode: recovered via httpx http2=True (was 403 on HTTP/1.1) - browser User-Agent + Accept headers on feed fetch - removed dead feeds with no active replacement: placeholder, dragonfly, niccarter, eugene - pin h2==4.3.0 (required by http2=True) All 25 remaining feeds verified fetching real body content; newest post per feed ≤88d. Bundles in-flight KOL-module work already in the working tree (kol_x ingest, migration 027, tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
267 lines
11 KiB
Python
267 lines
11 KiB
Python
"""
|
||
Generic signal ingestion endpoint.
|
||
|
||
The BTC bottom scanner POSTs a signal here and it flows through the shared
|
||
execution pipeline: sizing → risk caps → Hyperliquid execution → trailing
|
||
stop monitor. Unknown sources are accepted into the posts table for audit,
|
||
but the execution layer fails closed and will not trade them.
|
||
|
||
Why one endpoint instead of many?
|
||
The trusted scanners and the Trump scraper share the same trade execution
|
||
path. The downstream code sees a `Post` row with `signal=buy|short`, then
|
||
checks whether that post source is allowed to trade.
|
||
|
||
Auth: shared secret in X-Ingest-Key header. Fail-closed if INGEST_API_KEY
|
||
is empty in env (so a fresh deploy can't accidentally accept random POSTs).
|
||
|
||
Source field convention:
|
||
- "truth" → Trump posts (existing scraper). Reserved — rejected here.
|
||
- "btc_bottom_reversal" → Bitcoin Bottom scanner
|
||
- "<custom>" → Audit/storage only unless whitelisted in
|
||
signal_categories.py.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import logging
|
||
from datetime import datetime, timezone
|
||
from typing import Optional
|
||
|
||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||
from pydantic import BaseModel, Field, field_validator
|
||
from sqlalchemy import select
|
||
from sqlalchemy.exc import IntegrityError
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.config import settings
|
||
from app.database import get_db
|
||
from app.models import Post
|
||
|
||
router = APIRouter()
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ─── Schema ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
class SignalIngestRequest(BaseModel):
|
||
"""Payload for POST /api/signals/ingest.
|
||
|
||
Mirrors the Post schema closely so a signal here becomes a Post row
|
||
1:1, with no field translation. Downstream code is unaware of the source.
|
||
"""
|
||
|
||
source: str = Field(..., min_length=2, max_length=32,
|
||
description="Signal source tag, e.g. 'btc_bottom_reversal'. 'truth' is reserved.")
|
||
external_id: str = Field(..., min_length=4, max_length=64,
|
||
description="Caller-supplied unique key. Used for idempotent retries.")
|
||
text: str = Field(..., min_length=1, max_length=4000,
|
||
description="Human-readable description of why the signal fired. Shown in UI.")
|
||
signal: str = Field(..., description="'buy' or 'short' — non-actionable signals should not be ingested.")
|
||
target_asset: str = Field(..., min_length=2, max_length=16,
|
||
description="Hyperliquid perp ticker, e.g. 'SOL', 'BTC'.")
|
||
confidence: int = Field(..., ge=0, le=100,
|
||
description="0–100. Drives position sizing (≥90 boosts multiplier).")
|
||
category: str = Field(..., min_length=2, max_length=24,
|
||
description="Subtype within source, e.g. 'btc_bottom_reversal_long'.")
|
||
expected_move_pct: Optional[float] = Field(None, ge=0, le=100,
|
||
description="Optional: caller's expected % move. UI display only.")
|
||
invalidation_price: Optional[float] = Field(None, ge=0,
|
||
description="Optional thesis invalidation level for System 2.")
|
||
published_at: Optional[datetime] = Field(None,
|
||
description="Defaults to server now (UTC). Set explicitly only for backfill.")
|
||
|
||
@field_validator("signal")
|
||
@classmethod
|
||
def _signal_must_be_directional(cls, v: str) -> str:
|
||
if v not in ("buy", "short"):
|
||
raise ValueError(f"signal must be 'buy' or 'short' (got {v!r})")
|
||
return v
|
||
|
||
@field_validator("source")
|
||
@classmethod
|
||
def _no_reserved_sources(cls, v: str) -> str:
|
||
if v.lower() == "truth":
|
||
raise ValueError("'truth' is reserved for the Trump scraper. Use a different source tag.")
|
||
return v.lower()
|
||
|
||
|
||
class SignalIngestResponse(BaseModel):
|
||
status: str # "accepted" | "duplicate" | "skipped"
|
||
post_id: Optional[int] = None
|
||
dedup_against: Optional[int] = None
|
||
note: Optional[str] = None
|
||
|
||
|
||
# ─── Auth helper ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _verify_ingest_key(x_ingest_key: Optional[str]) -> None:
|
||
"""Fail-closed auth check.
|
||
|
||
- INGEST_API_KEY unset → return 503 (endpoint disabled by config)
|
||
- Header missing or wrong → 401
|
||
"""
|
||
expected = settings.ingest_api_key
|
||
if not expected:
|
||
raise HTTPException(503, "signal ingest endpoint is disabled (INGEST_API_KEY not configured)")
|
||
if not x_ingest_key:
|
||
raise HTTPException(401, "missing X-Ingest-Key header")
|
||
if x_ingest_key != expected:
|
||
raise HTTPException(401, "invalid X-Ingest-Key")
|
||
|
||
|
||
# ─── Endpoint ───────────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.post("/signals/ingest", response_model=SignalIngestResponse)
|
||
async def ingest_signal(
|
||
body: SignalIngestRequest,
|
||
x_ingest_key: Optional[str] = Header(None, alias="X-Ingest-Key"),
|
||
db: AsyncSession = Depends(get_db),
|
||
) -> SignalIngestResponse:
|
||
"""Accept a signal from an external trading module.
|
||
|
||
Pipeline behaviour:
|
||
1. Auth (shared secret in header)
|
||
2. Dedup by external_id (idempotent — same id returns the existing post_id)
|
||
3. Insert as a Post row (source = caller-supplied)
|
||
4. If the source is whitelisted for trading, hand off to
|
||
bot_engine.process_post. Otherwise store for audit and return
|
||
status="skipped".
|
||
|
||
Note: the entry filter ([A] in the pipeline) is SKIPPED for non-truth
|
||
sources. That filter is tuned for Trump's writing patterns; technical
|
||
signals are their own catalyst and don't need text-based gating.
|
||
"""
|
||
_verify_ingest_key(x_ingest_key)
|
||
|
||
# Hash external_id the same way truth_social.py does — keeps the dedup
|
||
# key short and uniform across sources.
|
||
ext_id_hashed = hashlib.md5(f"{body.source}:{body.external_id}".encode()).hexdigest()
|
||
|
||
# Idempotency check
|
||
existing = await db.execute(select(Post).where(Post.external_id == ext_id_hashed))
|
||
prior = existing.scalar_one_or_none()
|
||
if prior:
|
||
return SignalIngestResponse(
|
||
status="duplicate",
|
||
post_id=prior.id,
|
||
dedup_against=prior.id,
|
||
note=f"external_id {body.external_id!r} already ingested",
|
||
)
|
||
|
||
# Publication time defaults to server now (naive UTC, matching other rows)
|
||
pub_naive = (body.published_at or datetime.now(timezone.utc))
|
||
if pub_naive.tzinfo is not None:
|
||
pub_naive = pub_naive.astimezone(timezone.utc).replace(tzinfo=None)
|
||
|
||
post = Post(
|
||
external_id=ext_id_hashed,
|
||
text=body.text,
|
||
source=body.source,
|
||
published_at=pub_naive,
|
||
sentiment="bullish" if body.signal == "buy" else "bearish",
|
||
ai_confidence=body.confidence,
|
||
relevant=True,
|
||
signal=body.signal,
|
||
target_asset=body.target_asset.upper(),
|
||
category=body.category,
|
||
expected_move_pct=body.expected_move_pct,
|
||
invalidation_price=body.invalidation_price,
|
||
# `analysis_version` doubles as a provenance tag — easy to query in DB
|
||
analysis_version=f"ingest:{body.source}",
|
||
prefilter_reason="external_signal", # bypasses entry-filter audit
|
||
)
|
||
db.add(post)
|
||
try:
|
||
await db.commit()
|
||
await db.refresh(post)
|
||
except IntegrityError:
|
||
# Concurrent request beat us to the INSERT — fetch the existing row.
|
||
await db.rollback()
|
||
existing2 = await db.execute(select(Post).where(Post.external_id == ext_id_hashed))
|
||
prior2 = existing2.scalar_one_or_none()
|
||
if prior2:
|
||
return SignalIngestResponse(
|
||
status="duplicate",
|
||
post_id=prior2.id,
|
||
dedup_against=prior2.id,
|
||
note=f"concurrent ingest: external_id {body.external_id!r} already exists",
|
||
)
|
||
raise # unexpected — re-raise if we still can't find the row
|
||
|
||
logger.info(
|
||
"Ingested signal: source=%s id=%s → post_id=%d, %s/%s conf=%d category=%s",
|
||
body.source, body.external_id, post.id,
|
||
body.signal, body.target_asset, body.confidence, body.category,
|
||
)
|
||
|
||
from app.services.signal_categories import is_supported_trading_source
|
||
if not is_supported_trading_source(post.source):
|
||
logger.info("Signal %d stored but skipped: unsupported trading source=%s",
|
||
post.id, post.source)
|
||
return SignalIngestResponse(
|
||
status="skipped",
|
||
post_id=post.id,
|
||
note=f"source {post.source!r} is not enabled for live trading",
|
||
)
|
||
|
||
# Hand off to the same trade pipeline used by Trump posts.
|
||
# Runs sizing → risk caps → HL trade → trailing-stop monitor.
|
||
try:
|
||
from app.services.bot_engine import process_post
|
||
await process_post(post, db)
|
||
except Exception as exc:
|
||
# Don't fail the HTTP response — the post is already in DB, and
|
||
# process_post failures shouldn't make the external module retry
|
||
# (which would duplicate signals). Log and return success.
|
||
logger.error("process_post failed for ingested signal %d: %s", post.id, exc)
|
||
|
||
# Broadcast to UI so the new signal shows up live, same as Trump posts.
|
||
try:
|
||
from app.scrapers.truth_social import _post_to_ws_payload
|
||
from app.ws.manager import manager
|
||
await manager.broadcast(_post_to_ws_payload(post))
|
||
except Exception as exc:
|
||
logger.warning("WS broadcast failed for signal %d: %s", post.id, exc)
|
||
|
||
# Fan out to Telegram subscribers (fire-and-forget; never blocks ingest).
|
||
try:
|
||
from app.services.telegram import notify_signal
|
||
notify_signal(post)
|
||
except Exception as exc:
|
||
logger.warning("Telegram notify failed for signal %d: %s", post.id, exc)
|
||
|
||
# Auto-post a viral "live prediction" tweet (fire-and-forget; no-op unless
|
||
# X is configured + enabled). Internally gated to source=truth buy/short.
|
||
try:
|
||
from app.services.x_poster import notify_x_signal
|
||
notify_x_signal(post)
|
||
except Exception as exc:
|
||
logger.warning("X notify failed for signal %d: %s", post.id, exc)
|
||
|
||
return SignalIngestResponse(status="accepted", post_id=post.id)
|
||
|
||
|
||
@router.get("/signals/sources")
|
||
async def list_sources(db: AsyncSession = Depends(get_db)) -> dict:
|
||
"""List distinct signal sources we've seen, with counts.
|
||
|
||
Helpful for spot-checking that ingestion is working and for the UI to
|
||
show a source filter without hard-coding the list.
|
||
"""
|
||
from sqlalchemy import func
|
||
rows = await db.execute(
|
||
select(Post.source, func.count(Post.id), func.max(Post.published_at))
|
||
.group_by(Post.source)
|
||
.order_by(func.count(Post.id).desc())
|
||
)
|
||
return {
|
||
"sources": [
|
||
{"source": r[0], "count": r[1], "latest": r[2].isoformat() if r[2] else None}
|
||
for r in rows.all()
|
||
]
|
||
}
|