k
54884f3e24
Feed-health pass over KOL_FEEDS: - raoulpal: stale Substack (last 2024-05) → Real Vision podcast feed - dampedspring: paywalled (0 entries) → free "Damped Spring 101" Substack - unchained: Cloudflare 403 → canonical Megaphone podcast feed - lynalden: Cloudflare 202 → FeedBurner mirror - glassnode: recovered via httpx http2=True (was 403 on HTTP/1.1) - browser User-Agent + Accept headers on feed fetch - removed dead feeds with no active replacement: placeholder, dragonfly, niccarter, eugene - pin h2==4.3.0 (required by http2=True) All 25 remaining feeds verified fetching real body content; newest post per feed ≤88d. Bundles in-flight KOL-module work already in the working tree (kol_x ingest, migration 027, tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
438 lines
16 KiB
Python
438 lines
16 KiB
Python
"""KOL module — public read API.
|
|
|
|
Endpoints:
|
|
GET /api/kol/posts list KOL posts (newest first), with summary +
|
|
tickers but WITHOUT raw_text to keep payload small
|
|
GET /api/kol/posts/{id} full detail incl. raw_text for the modal/page view
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
from collections import defaultdict
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any, List, Optional
|
|
|
|
from fastapi import APIRouter, Depends, Header, HTTPException, Query
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.config import settings
|
|
from app.database import get_db
|
|
from app.models import KolDivergence, KolHoldingChange, KolHoldingSnapshot, KolPost, KolWallet, iso_utc
|
|
|
|
|
|
def _verify_admin_key(x_ingest_key: Optional[str]) -> None:
|
|
"""Shared-secret auth for write endpoints (wallet add, scan trigger).
|
|
Same key as the signal ingest endpoint. Fail-closed if INGEST_API_KEY unset."""
|
|
expected = settings.ingest_api_key
|
|
if not expected:
|
|
raise HTTPException(503, "write endpoint disabled (INGEST_API_KEY not configured)")
|
|
if not x_ingest_key:
|
|
raise HTTPException(401, "missing X-Ingest-Key header")
|
|
if x_ingest_key != expected:
|
|
raise HTTPException(401, "invalid X-Ingest-Key")
|
|
|
|
router = APIRouter()
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _parse_tickers(raw: Optional[str]) -> List[dict]:
|
|
if not raw:
|
|
return []
|
|
try:
|
|
data = json.loads(raw)
|
|
return data if isinstance(data, list) else []
|
|
except json.JSONDecodeError:
|
|
return []
|
|
|
|
|
|
def _summary_dto(post: KolPost) -> dict:
|
|
"""List-view shape: no raw_text, no content_hash."""
|
|
return {
|
|
"id": post.id,
|
|
"kol_handle": post.kol_handle,
|
|
"source": post.source,
|
|
"url": post.url,
|
|
"title": post.title,
|
|
"published_at": iso_utc(post.published_at),
|
|
"summary": post.summary,
|
|
"tickers": _parse_tickers(post.tickers_json),
|
|
"analyzed_at": iso_utc(post.analyzed_at),
|
|
"analysis_model": post.analysis_model,
|
|
# Extended x_analysis fields (migration 027)
|
|
"tier": post.tier,
|
|
"post_type": post.post_type,
|
|
"talks_vs_trades_flag": post.talks_vs_trades_flag or False,
|
|
"sentiment": post.sentiment,
|
|
}
|
|
|
|
|
|
def _detail_dto(post: KolPost) -> dict:
|
|
base = _summary_dto(post)
|
|
base["raw_text"] = post.raw_text
|
|
return base
|
|
|
|
|
|
@router.get("/kol/posts")
|
|
async def list_kol_posts(
|
|
handle: Optional[str] = Query(default=None, description="filter by kol_handle"),
|
|
source: Optional[str] = Query(default=None, description="substack | blog | podcast"),
|
|
signals_only: bool = Query(default=False,
|
|
description="exclude noise posts (tier='noise')"),
|
|
ticker: Optional[str] = Query(default=None, description="filter by ticker symbol e.g. BTC"),
|
|
days: Optional[int] = Query(default=None, ge=1, le=365, description="restrict to last N days"),
|
|
limit: int = Query(default=50, ge=1, le=200),
|
|
page: int = Query(default=1, ge=1),
|
|
db: AsyncSession = Depends(get_db),
|
|
) -> dict[str, Any]:
|
|
base = select(KolPost)
|
|
if handle:
|
|
base = base.where(KolPost.kol_handle == handle)
|
|
if source:
|
|
base = base.where(KolPost.source == source)
|
|
if signals_only:
|
|
base = base.where(
|
|
(KolPost.tier.is_(None)) | (KolPost.tier != "noise")
|
|
)
|
|
if ticker:
|
|
# tickers_json stores [{"ticker":"BTC",...}] — match the key-value pair
|
|
base = base.where(KolPost.tickers_json.like(f'%"ticker": "{ticker.upper()}"%'))
|
|
if days:
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
|
base = base.where(KolPost.published_at >= cutoff)
|
|
|
|
total: int = (await db.execute(
|
|
select(func.count()).select_from(base.subquery())
|
|
)).scalar_one()
|
|
|
|
rows = (await db.execute(
|
|
base.order_by(KolPost.published_at.desc())
|
|
.offset((page - 1) * limit)
|
|
.limit(limit)
|
|
)).scalars().all()
|
|
|
|
return {
|
|
"items": [_summary_dto(p) for p in rows],
|
|
"page": page,
|
|
"limit": limit,
|
|
"total": total,
|
|
}
|
|
|
|
|
|
@router.get("/kol/posts/{post_id}")
|
|
async def get_kol_post(post_id: int, db: AsyncSession = Depends(get_db)) -> dict:
|
|
row = (await db.execute(select(KolPost).where(KolPost.id == post_id))).scalar_one_or_none()
|
|
if row is None:
|
|
raise HTTPException(status_code=404, detail="KOL post not found")
|
|
return _detail_dto(row)
|
|
|
|
|
|
# ── Action ordering for picking the "dominant" call per ticker ─────────
|
|
# When the same ticker appears across multiple posts with different actions,
|
|
# the strongest signal wins. buy/sell (explicit position) outrank
|
|
# bullish/bearish (directional view) outrank mention (passing reference).
|
|
_ACTION_STRENGTH = {"buy": 3, "sell": 3, "bullish": 2, "bearish": 2, "mention": 1}
|
|
_ACTION_SIDE = {"buy": "long", "bullish": "long",
|
|
"sell": "short", "bearish": "short", "mention": "neutral"}
|
|
|
|
|
|
@router.get("/kol/digest")
|
|
async def kol_digest(
|
|
days: int = Query(default=7, ge=1, le=90,
|
|
description="rolling window in days"),
|
|
db: AsyncSession = Depends(get_db),
|
|
) -> dict[str, Any]:
|
|
"""Aggregate ticker calls across recent KOL posts.
|
|
|
|
For each ticker mentioned in posts within the window, returns the
|
|
dominant action, the highest conviction, and the list of (KOL, post,
|
|
action, conviction) calls. Sorted by post_count desc, then conviction.
|
|
"""
|
|
since = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days)
|
|
rows = (await db.execute(
|
|
select(KolPost)
|
|
.where(KolPost.published_at >= since)
|
|
.where(KolPost.tickers_json.is_not(None))
|
|
.order_by(KolPost.published_at.desc())
|
|
)).scalars().all()
|
|
|
|
# ticker -> list of call dicts
|
|
bucket: dict[str, list[dict]] = defaultdict(list)
|
|
for post in rows:
|
|
try:
|
|
tickers = json.loads(post.tickers_json or "[]")
|
|
except json.JSONDecodeError:
|
|
continue
|
|
for t in tickers:
|
|
sym = (t.get("ticker") or "").upper()
|
|
action = (t.get("action") or "mention").lower()
|
|
if not sym or action == "mention":
|
|
# Don't surface 'mention' on the digest — too noisy. Users
|
|
# can still see it on the post detail.
|
|
continue
|
|
bucket[sym].append({
|
|
"post_id": post.id,
|
|
"kol_handle": post.kol_handle,
|
|
"post_title": post.title,
|
|
"published_at": iso_utc(post.published_at),
|
|
"action": action,
|
|
"conviction": float(t.get("conviction") or 0.0),
|
|
"quote": (t.get("quote") or "")[:240],
|
|
})
|
|
|
|
tickers_out: list[dict] = []
|
|
for sym, calls in bucket.items():
|
|
# Dominant action: pick the action with the highest sum of conviction,
|
|
# broken by strength tier then count. Ensures one strong "buy" beats
|
|
# three weak "bullish" mentions.
|
|
action_weights: dict[str, float] = defaultdict(float)
|
|
for c in calls:
|
|
action_weights[c["action"]] += c["conviction"]
|
|
dominant = max(
|
|
action_weights.items(),
|
|
key=lambda kv: (_ACTION_STRENGTH.get(kv[0], 0), kv[1]),
|
|
)[0]
|
|
max_conv = max(c["conviction"] for c in calls)
|
|
unique_kols = sorted({c["kol_handle"] for c in calls})
|
|
tickers_out.append({
|
|
"ticker": sym,
|
|
"dominant_action": dominant,
|
|
"side": _ACTION_SIDE.get(dominant, "neutral"),
|
|
"max_conviction": round(max_conv, 2),
|
|
"post_count": len(calls),
|
|
"kol_count": len(unique_kols),
|
|
"kols": unique_kols,
|
|
"calls": sorted(calls, key=lambda c: c["conviction"], reverse=True),
|
|
})
|
|
|
|
tickers_out.sort(
|
|
key=lambda t: (t["kol_count"], t["post_count"], t["max_conviction"]),
|
|
reverse=True,
|
|
)
|
|
|
|
return {
|
|
"window_days": days,
|
|
"since": iso_utc(since),
|
|
"post_count": len(rows),
|
|
"ticker_count": len(tickers_out),
|
|
"tickers": tickers_out,
|
|
}
|
|
|
|
|
|
# ── A-tier: on-chain wallets + holdings ──────────────────────────────────────
|
|
|
|
def _wallet_dto(w: KolWallet) -> dict:
|
|
return {
|
|
"id": w.id,
|
|
"handle": w.handle,
|
|
"chain": w.chain,
|
|
"address": w.address,
|
|
"label": w.label,
|
|
"source_url": w.source_url,
|
|
"active": w.active,
|
|
"added_at": iso_utc(w.added_at),
|
|
}
|
|
|
|
|
|
def _snapshot_dto(s: KolHoldingSnapshot) -> dict:
|
|
return {
|
|
"id": s.id,
|
|
"wallet_id": s.wallet_id,
|
|
"snapshot_date": s.snapshot_date,
|
|
"holdings": json.loads(s.holdings_json or "[]"),
|
|
"total_usd": s.total_usd,
|
|
"source": s.source,
|
|
"created_at": iso_utc(s.created_at),
|
|
}
|
|
|
|
|
|
def _change_dto(c: KolHoldingChange, handle: str) -> dict:
|
|
return {
|
|
"id": c.id,
|
|
"wallet_id": c.wallet_id,
|
|
"handle": handle,
|
|
"detected_at": iso_utc(c.detected_at),
|
|
"ticker": c.ticker,
|
|
"change_type": c.change_type,
|
|
"usd_before": c.usd_before,
|
|
"usd_after": c.usd_after,
|
|
"pct_change": c.pct_change,
|
|
}
|
|
|
|
|
|
@router.get("/kol/wallets")
|
|
async def list_kol_wallets(
|
|
handle: Optional[str] = Query(default=None),
|
|
db: AsyncSession = Depends(get_db),
|
|
) -> dict[str, Any]:
|
|
stmt = select(KolWallet).where(KolWallet.active == True)
|
|
if handle:
|
|
stmt = stmt.where(KolWallet.handle == handle)
|
|
stmt = stmt.order_by(KolWallet.handle)
|
|
rows = (await db.execute(stmt)).scalars().all()
|
|
return {"wallets": [_wallet_dto(w) for w in rows]}
|
|
|
|
|
|
@router.post("/kol/wallets")
|
|
async def add_kol_wallet(
|
|
body: dict,
|
|
x_ingest_key: Optional[str] = Header(None, alias="X-Ingest-Key"),
|
|
db: AsyncSession = Depends(get_db),
|
|
) -> dict:
|
|
"""Add a new KOL wallet address. Body: {handle, chain, address, label?, source_url?}.
|
|
Auth: requires X-Ingest-Key header matching INGEST_API_KEY."""
|
|
_verify_admin_key(x_ingest_key)
|
|
required = {"handle", "chain", "address"}
|
|
if not required.issubset(body):
|
|
raise HTTPException(status_code=422, detail=f"Required fields: {required}")
|
|
|
|
# Check duplicate
|
|
existing = (await db.execute(
|
|
select(KolWallet).where(
|
|
KolWallet.chain == body["chain"],
|
|
KolWallet.address == body["address"].lower(),
|
|
)
|
|
)).scalar_one_or_none()
|
|
if existing:
|
|
raise HTTPException(status_code=409, detail="Wallet already tracked")
|
|
|
|
wallet = KolWallet(
|
|
handle=body["handle"],
|
|
chain=body["chain"],
|
|
address=body["address"].lower(),
|
|
label=body.get("label"),
|
|
source_url=body.get("source_url"),
|
|
)
|
|
db.add(wallet)
|
|
await db.commit()
|
|
await db.refresh(wallet)
|
|
return _wallet_dto(wallet)
|
|
|
|
|
|
@router.get("/kol/wallets/{wallet_id}/snapshots")
|
|
async def get_wallet_snapshots(
|
|
wallet_id: int,
|
|
limit: int = Query(default=30, ge=1, le=90),
|
|
db: AsyncSession = Depends(get_db),
|
|
) -> dict[str, Any]:
|
|
wallet = (await db.execute(
|
|
select(KolWallet).where(KolWallet.id == wallet_id)
|
|
)).scalar_one_or_none()
|
|
if not wallet:
|
|
raise HTTPException(status_code=404, detail="Wallet not found")
|
|
|
|
snaps = (await db.execute(
|
|
select(KolHoldingSnapshot)
|
|
.where(KolHoldingSnapshot.wallet_id == wallet_id)
|
|
.order_by(KolHoldingSnapshot.snapshot_date.desc())
|
|
.limit(limit)
|
|
)).scalars().all()
|
|
|
|
return {
|
|
"wallet": _wallet_dto(wallet),
|
|
"snapshots": [_snapshot_dto(s) for s in snaps],
|
|
}
|
|
|
|
|
|
@router.get("/kol/changes")
|
|
async def list_holding_changes(
|
|
handle: Optional[str] = Query(default=None),
|
|
days: int = Query(default=7, ge=1, le=90),
|
|
db: AsyncSession = Depends(get_db),
|
|
) -> dict[str, Any]:
|
|
"""Recent on-chain position changes across all tracked KOL wallets."""
|
|
since = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days)
|
|
|
|
stmt = (
|
|
select(KolHoldingChange, KolWallet.handle)
|
|
.join(KolWallet, KolHoldingChange.wallet_id == KolWallet.id)
|
|
.where(KolHoldingChange.detected_at >= since)
|
|
)
|
|
if handle:
|
|
stmt = stmt.where(KolWallet.handle == handle)
|
|
stmt = stmt.order_by(KolHoldingChange.detected_at.desc()).limit(200)
|
|
|
|
rows = (await db.execute(stmt)).all()
|
|
changes = [_change_dto(c, h) for c, h in rows]
|
|
return {
|
|
"window_days": days,
|
|
"since": iso_utc(since),
|
|
"count": len(changes),
|
|
"changes": changes,
|
|
}
|
|
|
|
|
|
# ── Talks-vs-trades divergence ────────────────────────────────────────────────
|
|
|
|
def _divergence_dto(d: KolDivergence) -> dict:
|
|
return {
|
|
"id": d.id,
|
|
"handle": d.handle,
|
|
"ticker": d.ticker,
|
|
"signal_type": d.signal_type, # divergence | alignment
|
|
"direction": d.direction, # long | short
|
|
"post_id": d.post_id,
|
|
"post_action": d.post_action,
|
|
"post_conviction":d.post_conviction,
|
|
"post_at": iso_utc(d.post_at),
|
|
"onchain_action": d.onchain_action,
|
|
"usd_before": d.usd_before,
|
|
"usd_after": d.usd_after,
|
|
"onchain_at": iso_utc(d.onchain_at),
|
|
"days_apart": d.days_apart,
|
|
"created_at": iso_utc(d.created_at),
|
|
}
|
|
|
|
|
|
@router.get("/kol/divergence")
|
|
async def list_divergence(
|
|
handle: Optional[str] = Query(default=None),
|
|
ticker: Optional[str] = Query(default=None),
|
|
signal_type: Optional[str] = Query(default=None, description="divergence | alignment"),
|
|
days: int = Query(default=30, ge=1, le=180),
|
|
db: AsyncSession = Depends(get_db),
|
|
) -> dict[str, Any]:
|
|
"""List talks-vs-trades cross-signal pairs.
|
|
|
|
signal_type=divergence → KOL said X but chain did the opposite (high alpha).
|
|
signal_type=alignment → KOL's words matched their on-chain action (reinforced signal).
|
|
"""
|
|
since = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days)
|
|
# Use post_at (when the KOL post was published) rather than created_at
|
|
# (when the divergence row was written to DB). created_at can lag by
|
|
# days or weeks when bulk scans are run, causing old event-pairs to
|
|
# appear as "recent" divergences long after they occurred.
|
|
stmt = select(KolDivergence).where(KolDivergence.post_at >= since)
|
|
if handle:
|
|
stmt = stmt.where(KolDivergence.handle == handle)
|
|
if ticker:
|
|
stmt = stmt.where(KolDivergence.ticker == ticker.upper())
|
|
if signal_type:
|
|
stmt = stmt.where(KolDivergence.signal_type == signal_type)
|
|
# Order by post_at (when the KOL actually published), consistent with the
|
|
# post_at window filter above. Ordering by created_at (DB write time) put
|
|
# a backfilled older post (post_at=05-18, created_at=05-28) ahead of a
|
|
# newer one (post_at=05-23), so the "latest divergence" read wrong.
|
|
# Tie-break on created_at so same-day posts have a stable order.
|
|
stmt = stmt.order_by(KolDivergence.post_at.desc(), KolDivergence.created_at.desc()).limit(200)
|
|
rows = (await db.execute(stmt)).scalars().all()
|
|
return {
|
|
"window_days": days,
|
|
"since": iso_utc(since),
|
|
"count": len(rows),
|
|
"items": [_divergence_dto(r) for r in rows],
|
|
}
|
|
|
|
|
|
@router.post("/kol/divergence/scan")
|
|
async def trigger_divergence_scan(
|
|
lookback_days: int = Query(default=30, ge=1, le=90),
|
|
x_ingest_key: Optional[str] = Header(None, alias="X-Ingest-Key"),
|
|
) -> dict[str, Any]:
|
|
"""Manually trigger the talks-vs-trades scan. Returns newly written pairs.
|
|
Auth: requires X-Ingest-Key header matching INGEST_API_KEY."""
|
|
_verify_admin_key(x_ingest_key)
|
|
from app.services.kol_divergence import run_divergence_scan
|
|
results = await run_divergence_scan(lookback_days=lookback_days)
|
|
return {"new_pairs": len(results), "items": results}
|