KOL feeds: fix dead/blocked sources, drop stale feeds (29→25)
Feed-health pass over KOL_FEEDS: - raoulpal: stale Substack (last 2024-05) → Real Vision podcast feed - dampedspring: paywalled (0 entries) → free "Damped Spring 101" Substack - unchained: Cloudflare 403 → canonical Megaphone podcast feed - lynalden: Cloudflare 202 → FeedBurner mirror - glassnode: recovered via httpx http2=True (was 403 on HTTP/1.1) - browser User-Agent + Accept headers on feed fetch - removed dead feeds with no active replacement: placeholder, dragonfly, niccarter, eugene - pin h2==4.3.0 (required by http2=True) All 25 remaining feeds verified fetching real body content; newest post per feed ≤88d. Bundles in-flight KOL-module work already in the working tree (kol_x ingest, migration 027, tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+52
-10
@@ -13,7 +13,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
@@ -59,6 +59,11 @@ def _summary_dto(post: KolPost) -> dict:
|
||||
"tickers": _parse_tickers(post.tickers_json),
|
||||
"analyzed_at": iso_utc(post.analyzed_at),
|
||||
"analysis_model": post.analysis_model,
|
||||
# Extended x_analysis fields (migration 027)
|
||||
"tier": post.tier,
|
||||
"post_type": post.post_type,
|
||||
"talks_vs_trades_flag": post.talks_vs_trades_flag or False,
|
||||
"sentiment": post.sentiment,
|
||||
}
|
||||
|
||||
|
||||
@@ -71,19 +76,47 @@ def _detail_dto(post: KolPost) -> dict:
|
||||
@router.get("/kol/posts")
|
||||
async def list_kol_posts(
|
||||
handle: Optional[str] = Query(default=None, description="filter by kol_handle"),
|
||||
source: Optional[str] = Query(default=None, description="substack | twitter"),
|
||||
source: Optional[str] = Query(default=None, description="substack | blog | podcast"),
|
||||
signals_only: bool = Query(default=False,
|
||||
description="exclude noise posts (tier='noise')"),
|
||||
ticker: Optional[str] = Query(default=None, description="filter by ticker symbol e.g. BTC"),
|
||||
days: Optional[int] = Query(default=None, ge=1, le=365, description="restrict to last N days"),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
page: int = Query(default=1, ge=1),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
stmt = select(KolPost)
|
||||
base = select(KolPost)
|
||||
if handle:
|
||||
stmt = stmt.where(KolPost.kol_handle == handle)
|
||||
base = base.where(KolPost.kol_handle == handle)
|
||||
if source:
|
||||
stmt = stmt.where(KolPost.source == source)
|
||||
stmt = stmt.order_by(KolPost.published_at.desc()).offset((page - 1) * limit).limit(limit)
|
||||
rows = (await db.execute(stmt)).scalars().all()
|
||||
return {"items": [_summary_dto(p) for p in rows], "page": page, "limit": limit}
|
||||
base = base.where(KolPost.source == source)
|
||||
if signals_only:
|
||||
base = base.where(
|
||||
(KolPost.tier.is_(None)) | (KolPost.tier != "noise")
|
||||
)
|
||||
if ticker:
|
||||
# tickers_json stores [{"ticker":"BTC",...}] — match the key-value pair
|
||||
base = base.where(KolPost.tickers_json.like(f'%"ticker": "{ticker.upper()}"%'))
|
||||
if days:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
base = base.where(KolPost.published_at >= cutoff)
|
||||
|
||||
total: int = (await db.execute(
|
||||
select(func.count()).select_from(base.subquery())
|
||||
)).scalar_one()
|
||||
|
||||
rows = (await db.execute(
|
||||
base.order_by(KolPost.published_at.desc())
|
||||
.offset((page - 1) * limit)
|
||||
.limit(limit)
|
||||
)).scalars().all()
|
||||
|
||||
return {
|
||||
"items": [_summary_dto(p) for p in rows],
|
||||
"page": page,
|
||||
"limit": limit,
|
||||
"total": total,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/kol/posts/{post_id}")
|
||||
@@ -365,14 +398,23 @@ async def list_divergence(
|
||||
signal_type=alignment → KOL's words matched their on-chain action (reinforced signal).
|
||||
"""
|
||||
since = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days)
|
||||
stmt = select(KolDivergence).where(KolDivergence.created_at >= since)
|
||||
# Use post_at (when the KOL post was published) rather than created_at
|
||||
# (when the divergence row was written to DB). created_at can lag by
|
||||
# days or weeks when bulk scans are run, causing old event-pairs to
|
||||
# appear as "recent" divergences long after they occurred.
|
||||
stmt = select(KolDivergence).where(KolDivergence.post_at >= since)
|
||||
if handle:
|
||||
stmt = stmt.where(KolDivergence.handle == handle)
|
||||
if ticker:
|
||||
stmt = stmt.where(KolDivergence.ticker == ticker.upper())
|
||||
if signal_type:
|
||||
stmt = stmt.where(KolDivergence.signal_type == signal_type)
|
||||
stmt = stmt.order_by(KolDivergence.created_at.desc()).limit(200)
|
||||
# Order by post_at (when the KOL actually published), consistent with the
|
||||
# post_at window filter above. Ordering by created_at (DB write time) put
|
||||
# a backfilled older post (post_at=05-18, created_at=05-28) ahead of a
|
||||
# newer one (post_at=05-23), so the "latest divergence" read wrong.
|
||||
# Tie-break on created_at so same-day posts have a stable order.
|
||||
stmt = stmt.order_by(KolDivergence.post_at.desc(), KolDivergence.created_at.desc()).limit(200)
|
||||
rows = (await db.execute(stmt)).scalars().all()
|
||||
return {
|
||||
"window_days": days,
|
||||
|
||||
Reference in New Issue
Block a user