KOL feeds: fix dead/blocked sources, drop stale feeds (29→25)
Feed-health pass over KOL_FEEDS: - raoulpal: stale Substack (last 2024-05) → Real Vision podcast feed - dampedspring: paywalled (0 entries) → free "Damped Spring 101" Substack - unchained: Cloudflare 403 → canonical Megaphone podcast feed - lynalden: Cloudflare 202 → FeedBurner mirror - glassnode: recovered via httpx http2=True (was 403 on HTTP/1.1) - browser User-Agent + Accept headers on feed fetch - removed dead feeds with no active replacement: placeholder, dragonfly, niccarter, eugene - pin h2==4.3.0 (required by http2=True) All 25 remaining feeds verified fetching real body content; newest post per feed ≤88d. Bundles in-flight KOL-module work already in the working tree (kol_x ingest, migration 027, tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
k
+167
-4
@@ -5,16 +5,29 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import Response
|
||||
|
||||
from app.ratelimit import limiter
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Post, iso_utc
|
||||
from app.schemas import PriceImpact, TrumpPost
|
||||
from app.schemas import PostFilterCounts, PostListResponse, PriceImpact, SourceCount, TrumpPost
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ARCHIVE_EXCLUDED_SOURCES = (
|
||||
"truth",
|
||||
"btc_bottom_reversal",
|
||||
"funding_reversal",
|
||||
"kol_divergence",
|
||||
)
|
||||
|
||||
|
||||
_AI_SCORED_EXPR = (
|
||||
(func.coalesce(Post.ai_confidence, 0) > 0) |
|
||||
Post.ai_reasoning.is_not(None)
|
||||
)
|
||||
|
||||
|
||||
def _direction_correct(signal: Optional[str], pct: Optional[float]) -> Optional[bool]:
|
||||
if pct is None or signal is None:
|
||||
@@ -98,11 +111,161 @@ async def get_posts(
|
||||
return [_post_to_schema(p) for p in posts]
|
||||
|
||||
|
||||
@router.get("/posts-paged", response_model=PostListResponse)
|
||||
@limiter.limit("60/minute")
|
||||
async def get_posts_page(
|
||||
request: Request,
|
||||
limit: int = Query(default=20, ge=1, le=500),
|
||||
page: int = Query(default=1, ge=1),
|
||||
source: Optional[str] = Query(
|
||||
default=None,
|
||||
description="Filter to a single source (e.g. 'truth', 'btc_bottom_reversal').",
|
||||
),
|
||||
source_in: Optional[str] = Query(
|
||||
default=None,
|
||||
description="Comma-separated allowlist of sources.",
|
||||
),
|
||||
source_not_in: Optional[str] = Query(
|
||||
default=None,
|
||||
description="Comma-separated denylist of sources.",
|
||||
),
|
||||
archive_only: bool = Query(
|
||||
default=False,
|
||||
description="When true, return only archived/retired sources (exclude live modules).",
|
||||
),
|
||||
sentiment: Optional[str] = Query(
|
||||
default=None,
|
||||
pattern="^(bullish|bearish|neutral)$",
|
||||
description="Optional sentiment filter.",
|
||||
),
|
||||
signal: Optional[str] = Query(
|
||||
default=None,
|
||||
pattern="^(buy|short|actionable)$",
|
||||
description="Optional signal filter. 'actionable' = buy or short.",
|
||||
),
|
||||
ai_scored_only: bool = Query(
|
||||
default=False,
|
||||
description="When true, exclude off-topic rows that were skipped before AI scoring.",
|
||||
),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
response: Response = None,
|
||||
):
|
||||
offset = (page - 1) * limit
|
||||
stmt = select(Post)
|
||||
count_stmt = select(func.count()).select_from(Post)
|
||||
counts_stmt = select(
|
||||
func.count().label("all_count"),
|
||||
func.sum(case((Post.signal.in_(("buy", "short")), 1), else_=0)).label("actionable_count"),
|
||||
func.sum(case((Post.signal == "buy", 1), else_=0)).label("buy_count"),
|
||||
func.sum(case((Post.signal == "short", 1), else_=0)).label("short_count"),
|
||||
func.sum(case((_AI_SCORED_EXPR, 0), else_=1)).label("off_topic_count"),
|
||||
).select_from(Post)
|
||||
source_counts_stmt = select(
|
||||
Post.source.label("source"),
|
||||
func.count(Post.id).label("count"),
|
||||
func.max(Post.published_at).label("latest"),
|
||||
).select_from(Post)
|
||||
|
||||
included_sources = [s.strip() for s in (source_in or "").split(",") if s.strip()]
|
||||
excluded_sources = [s.strip() for s in (source_not_in or "").split(",") if s.strip()]
|
||||
if archive_only:
|
||||
excluded_sources = list(dict.fromkeys([*excluded_sources, *_ARCHIVE_EXCLUDED_SOURCES]))
|
||||
|
||||
if source:
|
||||
stmt = stmt.where(Post.source == source)
|
||||
count_stmt = count_stmt.where(Post.source == source)
|
||||
counts_stmt = counts_stmt.where(Post.source == source)
|
||||
source_counts_stmt = source_counts_stmt.where(Post.source == source)
|
||||
elif included_sources:
|
||||
stmt = stmt.where(Post.source.in_(included_sources))
|
||||
count_stmt = count_stmt.where(Post.source.in_(included_sources))
|
||||
counts_stmt = counts_stmt.where(Post.source.in_(included_sources))
|
||||
# NOTE: source_counts is the chip/source breakdown the UI renders the
|
||||
# filter bar from. It must reflect every source available in the
|
||||
# current view scope — NOT just the one the user has selected. So
|
||||
# `source_in` (the chip-selection narrowing) is deliberately NOT
|
||||
# applied here; only the exclusion filters (archive_only /
|
||||
# source_not_in) below scope it. Applying it would collapse the chip
|
||||
# bar to the single selected source with no way back to "all".
|
||||
|
||||
if excluded_sources:
|
||||
stmt = stmt.where(~Post.source.in_(excluded_sources))
|
||||
count_stmt = count_stmt.where(~Post.source.in_(excluded_sources))
|
||||
counts_stmt = counts_stmt.where(~Post.source.in_(excluded_sources))
|
||||
source_counts_stmt = source_counts_stmt.where(~Post.source.in_(excluded_sources))
|
||||
|
||||
if sentiment:
|
||||
stmt = stmt.where(Post.sentiment == sentiment)
|
||||
count_stmt = count_stmt.where(Post.sentiment == sentiment)
|
||||
counts_stmt = counts_stmt.where(Post.sentiment == sentiment)
|
||||
source_counts_stmt = source_counts_stmt.where(Post.sentiment == sentiment)
|
||||
if ai_scored_only:
|
||||
stmt = stmt.where(_AI_SCORED_EXPR)
|
||||
count_stmt = count_stmt.where(_AI_SCORED_EXPR)
|
||||
source_counts_stmt = source_counts_stmt.where(_AI_SCORED_EXPR)
|
||||
|
||||
if signal == "actionable":
|
||||
stmt = stmt.where(Post.signal.in_(("buy", "short")))
|
||||
count_stmt = count_stmt.where(Post.signal.in_(("buy", "short")))
|
||||
elif signal:
|
||||
stmt = stmt.where(Post.signal == signal)
|
||||
count_stmt = count_stmt.where(Post.signal == signal)
|
||||
|
||||
stmt = stmt.order_by(Post.published_at.desc()).offset(offset).limit(limit)
|
||||
result = await db.execute(stmt)
|
||||
total_result = await db.execute(count_stmt)
|
||||
counts_result = await db.execute(counts_stmt)
|
||||
source_counts_result = await db.execute(
|
||||
source_counts_stmt.group_by(Post.source).order_by(func.count(Post.id).desc(), Post.source.asc())
|
||||
)
|
||||
posts = result.scalars().all()
|
||||
total = int(total_result.scalar_one() or 0)
|
||||
counts_row = counts_result.one()
|
||||
off_topic = int(counts_row.off_topic_count or 0)
|
||||
all_count = int(counts_row.all_count or 0)
|
||||
if response is not None:
|
||||
response.headers["Cache-Control"] = "public, max-age=30, stale-while-revalidate=60"
|
||||
return PostListResponse(
|
||||
items=[_post_to_schema(p) for p in posts],
|
||||
total=total,
|
||||
page=page,
|
||||
limit=limit,
|
||||
counts=PostFilterCounts(
|
||||
all=max(0, all_count - off_topic) if ai_scored_only else all_count,
|
||||
actionable=int(counts_row.actionable_count or 0),
|
||||
buy=int(counts_row.buy_count or 0),
|
||||
short=int(counts_row.short_count or 0),
|
||||
off_topic=off_topic,
|
||||
),
|
||||
source_counts=[
|
||||
SourceCount(
|
||||
source=row.source,
|
||||
count=int(row.count or 0),
|
||||
latest=iso_utc(row.latest),
|
||||
)
|
||||
for row in source_counts_result.all()
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@router.get("/signals/accuracy")
|
||||
async def signal_accuracy(db: AsyncSession = Depends(get_db)):
|
||||
"""Aggregate accuracy of directional signals (buy/sell/short) against realised price moves."""
|
||||
"""Aggregate accuracy of directional signals against realised price moves.
|
||||
|
||||
Scoped to the CURRENT signal taxonomy the live bot actually trades:
|
||||
* only buy/short (the retired "sell" vocabulary is excluded — the bot
|
||||
emits buy/short now, and 42 legacy truth/sell rows would otherwise
|
||||
pollute the public scoreboard),
|
||||
* only production sources (SUPPORTED_TRADING_SOURCES) — retired/test
|
||||
ingest sources like rsi_reversal, sma_reclaim, breakout, phase1 and
|
||||
`test` must not appear in the public accuracy stats.
|
||||
"""
|
||||
from app.services.signal_categories import SUPPORTED_TRADING_SOURCES
|
||||
result = await db.execute(
|
||||
select(Post).where(Post.signal.in_(["buy", "sell", "short"]))
|
||||
select(Post).where(
|
||||
Post.signal.in_(["buy", "short"]),
|
||||
func.lower(Post.source).in_(SUPPORTED_TRADING_SOURCES),
|
||||
)
|
||||
)
|
||||
posts = result.scalars().all()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user