k
54884f3e24
Feed-health pass over KOL_FEEDS: - raoulpal: stale Substack (last 2024-05) → Real Vision podcast feed - dampedspring: paywalled (0 entries) → free "Damped Spring 101" Substack - unchained: Cloudflare 403 → canonical Megaphone podcast feed - lynalden: Cloudflare 202 → FeedBurner mirror - glassnode: recovered via httpx http2=True (was 403 on HTTP/1.1) - browser User-Agent + Accept headers on feed fetch - removed dead feeds with no active replacement: placeholder, dragonfly, niccarter, eugene - pin h2==4.3.0 (required by http2=True) All 25 remaining feeds verified fetching real body content; newest post per feed ≤88d. Bundles in-flight KOL-module work already in the working tree (kol_x ingest, migration 027, tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
316 lines
12 KiB
Python
316 lines
12 KiB
Python
import logging
|
|
from typing import List, Optional
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
|
from fastapi.responses import Response
|
|
|
|
from app.ratelimit import limiter
|
|
from sqlalchemy import case, func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.database import get_db
|
|
from app.models import Post, iso_utc
|
|
from app.schemas import PostFilterCounts, PostListResponse, PriceImpact, SourceCount, TrumpPost
|
|
|
|
router = APIRouter()
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_ARCHIVE_EXCLUDED_SOURCES = (
|
|
"truth",
|
|
"btc_bottom_reversal",
|
|
"funding_reversal",
|
|
"kol_divergence",
|
|
)
|
|
|
|
|
|
_AI_SCORED_EXPR = (
|
|
(func.coalesce(Post.ai_confidence, 0) > 0) |
|
|
Post.ai_reasoning.is_not(None)
|
|
)
|
|
|
|
|
|
def _direction_correct(signal: Optional[str], pct: Optional[float]) -> Optional[bool]:
|
|
if pct is None or signal is None:
|
|
return None
|
|
if signal == "buy":
|
|
return pct > 0
|
|
if signal in ("short", "sell"):
|
|
return pct < 0
|
|
return None # hold has no direction
|
|
|
|
|
|
def _post_to_schema(post: Post) -> TrumpPost:
|
|
price_impact: Optional[PriceImpact] = None
|
|
if post.price_impact_asset and post.price_at_post is not None:
|
|
# Overlay live rolling peaks for windows that haven't closed yet
|
|
from app.services.price_impact_monitor import get_live_impact
|
|
live = get_live_impact(post.id) or {}
|
|
|
|
m5 = live.get("price_impact_m5", post.price_impact_m5)
|
|
m15 = live.get("price_impact_m15", post.price_impact_m15)
|
|
m1h = live.get("price_impact_m1h", post.price_impact_m1h)
|
|
|
|
price_impact = PriceImpact(
|
|
asset=post.price_impact_asset,
|
|
m5=m5,
|
|
m15=m15,
|
|
m1h=m1h,
|
|
price_at_post=post.price_at_post,
|
|
correct_m5=_direction_correct(post.signal, m5),
|
|
correct_m15=_direction_correct(post.signal, m15),
|
|
correct_m1h=_direction_correct(post.signal, m1h),
|
|
)
|
|
return TrumpPost(
|
|
id=post.id,
|
|
text=post.text,
|
|
source=post.source,
|
|
published_at=iso_utc(post.published_at),
|
|
sentiment=post.sentiment,
|
|
signal=post.signal,
|
|
ai_confidence=post.ai_confidence,
|
|
ai_reasoning=post.ai_reasoning,
|
|
prefilter_reason=post.prefilter_reason,
|
|
analysis_version=post.analysis_version,
|
|
relevant=post.relevant,
|
|
price_impact=price_impact,
|
|
# v5 routing fields — null for pre-v5 posts
|
|
target_asset=post.target_asset,
|
|
category=post.category,
|
|
expected_move_pct=post.expected_move_pct,
|
|
invalidation_price=post.invalidation_price,
|
|
)
|
|
|
|
|
|
@router.get("/posts", response_model=List[TrumpPost])
|
|
@limiter.limit("60/minute")
|
|
async def get_posts(
|
|
request: Request,
|
|
limit: int = Query(default=20, ge=1, le=500),
|
|
page: int = Query(default=1, ge=1),
|
|
source: Optional[str] = Query(
|
|
default=None,
|
|
description="Filter to a single source (e.g. 'btc_bottom_reversal', "
|
|
"'funding_reversal', 'truth'). Without it, rare-source "
|
|
"signals can be pushed off the latest-N page by Trump posts.",
|
|
),
|
|
db: AsyncSession = Depends(get_db),
|
|
response: Response = None,
|
|
):
|
|
offset = (page - 1) * limit
|
|
stmt = select(Post)
|
|
if source:
|
|
stmt = stmt.where(Post.source == source)
|
|
stmt = stmt.order_by(Post.published_at.desc()).offset(offset).limit(limit)
|
|
result = await db.execute(stmt)
|
|
posts = result.scalars().all()
|
|
# Posts are scraped every 5s but rarely change once written — allow CDN/browser
|
|
# to cache for 30s. stale-while-revalidate=60 means stale content is served
|
|
# while a fresh fetch happens in the background (no loading flash).
|
|
if response is not None:
|
|
response.headers["Cache-Control"] = "public, max-age=30, stale-while-revalidate=60"
|
|
return [_post_to_schema(p) for p in posts]
|
|
|
|
|
|
@router.get("/posts-paged", response_model=PostListResponse)
|
|
@limiter.limit("60/minute")
|
|
async def get_posts_page(
|
|
request: Request,
|
|
limit: int = Query(default=20, ge=1, le=500),
|
|
page: int = Query(default=1, ge=1),
|
|
source: Optional[str] = Query(
|
|
default=None,
|
|
description="Filter to a single source (e.g. 'truth', 'btc_bottom_reversal').",
|
|
),
|
|
source_in: Optional[str] = Query(
|
|
default=None,
|
|
description="Comma-separated allowlist of sources.",
|
|
),
|
|
source_not_in: Optional[str] = Query(
|
|
default=None,
|
|
description="Comma-separated denylist of sources.",
|
|
),
|
|
archive_only: bool = Query(
|
|
default=False,
|
|
description="When true, return only archived/retired sources (exclude live modules).",
|
|
),
|
|
sentiment: Optional[str] = Query(
|
|
default=None,
|
|
pattern="^(bullish|bearish|neutral)$",
|
|
description="Optional sentiment filter.",
|
|
),
|
|
signal: Optional[str] = Query(
|
|
default=None,
|
|
pattern="^(buy|short|actionable)$",
|
|
description="Optional signal filter. 'actionable' = buy or short.",
|
|
),
|
|
ai_scored_only: bool = Query(
|
|
default=False,
|
|
description="When true, exclude off-topic rows that were skipped before AI scoring.",
|
|
),
|
|
db: AsyncSession = Depends(get_db),
|
|
response: Response = None,
|
|
):
|
|
offset = (page - 1) * limit
|
|
stmt = select(Post)
|
|
count_stmt = select(func.count()).select_from(Post)
|
|
counts_stmt = select(
|
|
func.count().label("all_count"),
|
|
func.sum(case((Post.signal.in_(("buy", "short")), 1), else_=0)).label("actionable_count"),
|
|
func.sum(case((Post.signal == "buy", 1), else_=0)).label("buy_count"),
|
|
func.sum(case((Post.signal == "short", 1), else_=0)).label("short_count"),
|
|
func.sum(case((_AI_SCORED_EXPR, 0), else_=1)).label("off_topic_count"),
|
|
).select_from(Post)
|
|
source_counts_stmt = select(
|
|
Post.source.label("source"),
|
|
func.count(Post.id).label("count"),
|
|
func.max(Post.published_at).label("latest"),
|
|
).select_from(Post)
|
|
|
|
included_sources = [s.strip() for s in (source_in or "").split(",") if s.strip()]
|
|
excluded_sources = [s.strip() for s in (source_not_in or "").split(",") if s.strip()]
|
|
if archive_only:
|
|
excluded_sources = list(dict.fromkeys([*excluded_sources, *_ARCHIVE_EXCLUDED_SOURCES]))
|
|
|
|
if source:
|
|
stmt = stmt.where(Post.source == source)
|
|
count_stmt = count_stmt.where(Post.source == source)
|
|
counts_stmt = counts_stmt.where(Post.source == source)
|
|
source_counts_stmt = source_counts_stmt.where(Post.source == source)
|
|
elif included_sources:
|
|
stmt = stmt.where(Post.source.in_(included_sources))
|
|
count_stmt = count_stmt.where(Post.source.in_(included_sources))
|
|
counts_stmt = counts_stmt.where(Post.source.in_(included_sources))
|
|
# NOTE: source_counts is the chip/source breakdown the UI renders the
|
|
# filter bar from. It must reflect every source available in the
|
|
# current view scope — NOT just the one the user has selected. So
|
|
# `source_in` (the chip-selection narrowing) is deliberately NOT
|
|
# applied here; only the exclusion filters (archive_only /
|
|
# source_not_in) below scope it. Applying it would collapse the chip
|
|
# bar to the single selected source with no way back to "all".
|
|
|
|
if excluded_sources:
|
|
stmt = stmt.where(~Post.source.in_(excluded_sources))
|
|
count_stmt = count_stmt.where(~Post.source.in_(excluded_sources))
|
|
counts_stmt = counts_stmt.where(~Post.source.in_(excluded_sources))
|
|
source_counts_stmt = source_counts_stmt.where(~Post.source.in_(excluded_sources))
|
|
|
|
if sentiment:
|
|
stmt = stmt.where(Post.sentiment == sentiment)
|
|
count_stmt = count_stmt.where(Post.sentiment == sentiment)
|
|
counts_stmt = counts_stmt.where(Post.sentiment == sentiment)
|
|
source_counts_stmt = source_counts_stmt.where(Post.sentiment == sentiment)
|
|
if ai_scored_only:
|
|
stmt = stmt.where(_AI_SCORED_EXPR)
|
|
count_stmt = count_stmt.where(_AI_SCORED_EXPR)
|
|
source_counts_stmt = source_counts_stmt.where(_AI_SCORED_EXPR)
|
|
|
|
if signal == "actionable":
|
|
stmt = stmt.where(Post.signal.in_(("buy", "short")))
|
|
count_stmt = count_stmt.where(Post.signal.in_(("buy", "short")))
|
|
elif signal:
|
|
stmt = stmt.where(Post.signal == signal)
|
|
count_stmt = count_stmt.where(Post.signal == signal)
|
|
|
|
stmt = stmt.order_by(Post.published_at.desc()).offset(offset).limit(limit)
|
|
result = await db.execute(stmt)
|
|
total_result = await db.execute(count_stmt)
|
|
counts_result = await db.execute(counts_stmt)
|
|
source_counts_result = await db.execute(
|
|
source_counts_stmt.group_by(Post.source).order_by(func.count(Post.id).desc(), Post.source.asc())
|
|
)
|
|
posts = result.scalars().all()
|
|
total = int(total_result.scalar_one() or 0)
|
|
counts_row = counts_result.one()
|
|
off_topic = int(counts_row.off_topic_count or 0)
|
|
all_count = int(counts_row.all_count or 0)
|
|
if response is not None:
|
|
response.headers["Cache-Control"] = "public, max-age=30, stale-while-revalidate=60"
|
|
return PostListResponse(
|
|
items=[_post_to_schema(p) for p in posts],
|
|
total=total,
|
|
page=page,
|
|
limit=limit,
|
|
counts=PostFilterCounts(
|
|
all=max(0, all_count - off_topic) if ai_scored_only else all_count,
|
|
actionable=int(counts_row.actionable_count or 0),
|
|
buy=int(counts_row.buy_count or 0),
|
|
short=int(counts_row.short_count or 0),
|
|
off_topic=off_topic,
|
|
),
|
|
source_counts=[
|
|
SourceCount(
|
|
source=row.source,
|
|
count=int(row.count or 0),
|
|
latest=iso_utc(row.latest),
|
|
)
|
|
for row in source_counts_result.all()
|
|
],
|
|
)
|
|
|
|
|
|
@router.get("/signals/accuracy")
|
|
async def signal_accuracy(db: AsyncSession = Depends(get_db)):
|
|
"""Aggregate accuracy of directional signals against realised price moves.
|
|
|
|
Scoped to the CURRENT signal taxonomy the live bot actually trades:
|
|
* only buy/short (the retired "sell" vocabulary is excluded — the bot
|
|
emits buy/short now, and 42 legacy truth/sell rows would otherwise
|
|
pollute the public scoreboard),
|
|
* only production sources (SUPPORTED_TRADING_SOURCES) — retired/test
|
|
ingest sources like rsi_reversal, sma_reclaim, breakout, phase1 and
|
|
`test` must not appear in the public accuracy stats.
|
|
"""
|
|
from app.services.signal_categories import SUPPORTED_TRADING_SOURCES
|
|
result = await db.execute(
|
|
select(Post).where(
|
|
Post.signal.in_(["buy", "short"]),
|
|
func.lower(Post.source).in_(SUPPORTED_TRADING_SOURCES),
|
|
)
|
|
)
|
|
posts = result.scalars().all()
|
|
|
|
def bucket():
|
|
return {"checked": 0, "correct": 0}
|
|
|
|
stats = {"m5": bucket(), "m15": bucket(), "m1h": bucket()}
|
|
by_signal: dict[str, dict] = {}
|
|
|
|
for p in posts:
|
|
sig = p.signal
|
|
if sig not in by_signal:
|
|
by_signal[sig] = {"m5": bucket(), "m15": bucket(), "m1h": bucket(), "count": 0}
|
|
by_signal[sig]["count"] += 1
|
|
for win, val in (("m5", p.price_impact_m5), ("m15", p.price_impact_m15), ("m1h", p.price_impact_m1h)):
|
|
ok = _direction_correct(sig, val)
|
|
if ok is None:
|
|
continue
|
|
stats[win]["checked"] += 1
|
|
stats[win]["correct"] += int(ok)
|
|
by_signal[sig][win]["checked"] += 1
|
|
by_signal[sig][win]["correct"] += int(ok)
|
|
|
|
def pct(b): return round(b["correct"] / b["checked"] * 100, 1) if b["checked"] else None
|
|
|
|
return {
|
|
"overall": {k: {**v, "accuracy_pct": pct(v)} for k, v in stats.items()},
|
|
"by_signal": {
|
|
s: {
|
|
"count": d["count"],
|
|
"m5": {**d["m5"], "accuracy_pct": pct(d["m5"])},
|
|
"m15": {**d["m15"], "accuracy_pct": pct(d["m15"])},
|
|
"m1h": {**d["m1h"], "accuracy_pct": pct(d["m1h"])},
|
|
}
|
|
for s, d in by_signal.items()
|
|
},
|
|
"total_directional_signals": len(posts),
|
|
}
|
|
|
|
|
|
@router.get("/posts/{post_id}", response_model=TrumpPost)
|
|
async def get_post(post_id: int, db: AsyncSession = Depends(get_db)):
|
|
result = await db.execute(select(Post).where(Post.id == post_id))
|
|
post = result.scalar_one_or_none()
|
|
if post is None:
|
|
raise HTTPException(status_code=404, detail="Post not found")
|
|
return _post_to_schema(post)
|