KOL feeds: fix dead/blocked sources, drop stale feeds (29→25)

Feed-health pass over KOL_FEEDS:
- raoulpal: stale Substack (last 2024-05) → Real Vision podcast feed
- dampedspring: paywalled (0 entries) → free "Damped Spring 101" Substack
- unchained: Cloudflare 403 → canonical Megaphone podcast feed
- lynalden: Cloudflare 202 → FeedBurner mirror
- glassnode: recovered via httpx http2=True (was 403 on HTTP/1.1)
- browser User-Agent + Accept headers on feed fetch
- removed dead feeds with no active replacement: placeholder,
  dragonfly, niccarter, eugene
- pin h2==4.3.0 (required by http2=True)

All 25 remaining feeds verified fetching real body content; newest
post per feed ≤88d. Bundles in-flight KOL-module work already in the
working tree (kol_x ingest, migration 027, tests).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
k
2026-06-09 22:55:16 +08:00
parent 213bb911e3
commit 54884f3e24
38 changed files with 2340 additions and 322 deletions
+52 -10
View File
@@ -13,7 +13,7 @@ from datetime import datetime, timedelta, timezone
from typing import Any, List, Optional
from fastapi import APIRouter, Depends, Header, HTTPException, Query
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
@@ -59,6 +59,11 @@ def _summary_dto(post: KolPost) -> dict:
"tickers": _parse_tickers(post.tickers_json),
"analyzed_at": iso_utc(post.analyzed_at),
"analysis_model": post.analysis_model,
# Extended x_analysis fields (migration 027)
"tier": post.tier,
"post_type": post.post_type,
"talks_vs_trades_flag": post.talks_vs_trades_flag or False,
"sentiment": post.sentiment,
}
@@ -71,19 +76,47 @@ def _detail_dto(post: KolPost) -> dict:
@router.get("/kol/posts")
async def list_kol_posts(
handle: Optional[str] = Query(default=None, description="filter by kol_handle"),
source: Optional[str] = Query(default=None, description="substack | twitter"),
source: Optional[str] = Query(default=None, description="substack | blog | podcast"),
signals_only: bool = Query(default=False,
description="exclude noise posts (tier='noise')"),
ticker: Optional[str] = Query(default=None, description="filter by ticker symbol e.g. BTC"),
days: Optional[int] = Query(default=None, ge=1, le=365, description="restrict to last N days"),
limit: int = Query(default=50, ge=1, le=200),
page: int = Query(default=1, ge=1),
db: AsyncSession = Depends(get_db),
) -> dict[str, Any]:
stmt = select(KolPost)
base = select(KolPost)
if handle:
stmt = stmt.where(KolPost.kol_handle == handle)
base = base.where(KolPost.kol_handle == handle)
if source:
stmt = stmt.where(KolPost.source == source)
stmt = stmt.order_by(KolPost.published_at.desc()).offset((page - 1) * limit).limit(limit)
rows = (await db.execute(stmt)).scalars().all()
return {"items": [_summary_dto(p) for p in rows], "page": page, "limit": limit}
base = base.where(KolPost.source == source)
if signals_only:
base = base.where(
(KolPost.tier.is_(None)) | (KolPost.tier != "noise")
)
if ticker:
# tickers_json stores [{"ticker":"BTC",...}] — match the key-value pair
base = base.where(KolPost.tickers_json.like(f'%"ticker": "{ticker.upper()}"%'))
if days:
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
base = base.where(KolPost.published_at >= cutoff)
total: int = (await db.execute(
select(func.count()).select_from(base.subquery())
)).scalar_one()
rows = (await db.execute(
base.order_by(KolPost.published_at.desc())
.offset((page - 1) * limit)
.limit(limit)
)).scalars().all()
return {
"items": [_summary_dto(p) for p in rows],
"page": page,
"limit": limit,
"total": total,
}
@router.get("/kol/posts/{post_id}")
@@ -365,14 +398,23 @@ async def list_divergence(
signal_type=alignment → KOL's words matched their on-chain action (reinforced signal).
"""
since = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days)
stmt = select(KolDivergence).where(KolDivergence.created_at >= since)
# Use post_at (when the KOL post was published) rather than created_at
# (when the divergence row was written to DB). created_at can lag by
# days or weeks when bulk scans are run, causing old event-pairs to
# appear as "recent" divergences long after they occurred.
stmt = select(KolDivergence).where(KolDivergence.post_at >= since)
if handle:
stmt = stmt.where(KolDivergence.handle == handle)
if ticker:
stmt = stmt.where(KolDivergence.ticker == ticker.upper())
if signal_type:
stmt = stmt.where(KolDivergence.signal_type == signal_type)
stmt = stmt.order_by(KolDivergence.created_at.desc()).limit(200)
# Order by post_at (when the KOL actually published), consistent with the
# post_at window filter above. Ordering by created_at (DB write time) put
# a backfilled older post (post_at=05-18, created_at=05-28) ahead of a
# newer one (post_at=05-23), so the "latest divergence" read wrong.
# Tie-break on created_at so same-day posts have a stable order.
stmt = stmt.order_by(KolDivergence.post_at.desc(), KolDivergence.created_at.desc()).limit(200)
rows = (await db.execute(stmt)).scalars().all()
return {
"window_days": days,