"""KOL Substack RSS ingester. Polls each tracked KOL's Substack feed, dedupes by URL, stores raw post, then hands off to kol_analysis.extract_kol_signal and writes the result back onto the same row. Substack RSS embeds the full post HTML in . We strip HTML to plain text before storage + analysis. Hayes posts are typically 50K+ chars of body — the extractor truncates internally. Daily cadence is plenty (Hayes posts ~monthly, Substack updates feed within minutes of publish). Call run_substack_poll() from the APScheduler. """ from __future__ import annotations import hashlib import logging import re from datetime import datetime, timezone from email.utils import parsedate_to_datetime from typing import Iterable, Optional import feedparser import httpx from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.database import AsyncSessionLocal from app.models import KolPost, utcnow from app.services import kol_analysis logger = logging.getLogger(__name__) # Curated B-tier KOL feeds. Handle is the canonical key. `source` is the # DB column ("substack" | "podcast" | "blog"); empty defaults to "substack" # for legacy entries. Twitter-only KOLs come in a separate ingester. # # When adding a new feed: # 1. curl + grep '' to confirm it returns entries. # 2. Inspect entry summary/content length — AI extraction needs ≥300 chars # of body per post or it just hallucinates a topic line. (Headlines-only # feeds like Vitalik's blog need a follow-up HTML fetch, deferred.) # 3. Add with a sensible handle + display_name. KOL_FEEDS: list[dict] = [ # ── Substack essayists (long-form thesis pieces) ───────────────────── { "handle": "cryptohayes", "display_name": "Arthur Hayes", "feed_url": "https://cryptohayes.substack.com/feed", }, # Placeholder VC (Joel Monegro / Chris Burniske). Token-focused VC, posts # long-form thesis pieces every 1-3 months that map directly to their # portfolio bets (Solana staking, L1 monetary premium, etc.). { "handle": "placeholder", "display_name": "Placeholder VC", "feed_url": "https://www.placeholder.vc/blog?format=rss", }, # Dragonfly Capital research blog on Medium — free, active (10+ posts). # dragonfly.xyz/blog/rss.xml returns 0 (paywall). medium.com/dragonfly-research # is the team's public research arm: airdrops, DeFi, protocol deep-dives. { "handle": "dragonfly", "display_name": "Dragonfly Capital", "feed_url": "https://medium.com/feed/dragonfly-research", }, # Andy Constan's Substack is paywalled (RSS returns 0). Keeping for any # occasional public teaser. Forward Guidance podcast (Blockworks) features # him weekly but is macro/equities-focused — not crypto-coin-specific enough # to extract ticker signals from episode descriptions. { "handle": "dampedspring", "display_name": "Damped Spring / Andy Constan", "feed_url": "https://dampedspring.substack.com/feed", }, # Nic Carter's Substack is paywalled (RSS returns 0). His Medium feed is # FREE and active — different URL, same author, real content. { "handle": "niccarter", "display_name": "Nic Carter (Castle Island)", "feed_url": "https://medium.com/feed/@nic__carter", }, # Delphi Digital podcast (Buzzsprout) — 478 episodes, active May 2025. # Public, free. Episode descriptions name specific protocols / tokens with # thesis framing — good extraction signal. delphidigital.io/feed returns 0. { "handle": "delphi", "display_name": "Delphi Digital (Podcast)", "feed_url": "https://rss.buzzsprout.com/2609274.rss", }, # ── Newly added (verified live + active) ───────────────────────────── # Anthony Pompliano — Pomp Investments. Active monthly+ on macro/crypto. { "handle": "pomp", "display_name": "Anthony Pompliano (Pomp Letter)", "feed_url": "https://pomp.substack.com/feed", }, # The DeFi Edge — researcher who writes 1-2 deep dives per month on # tokens / sectors. Real thesis + position-aware framing. { "handle": "thedefiedge", "display_name": "The DeFi Edge", "feed_url": "https://thedefiedge.com/feed/", }, # Eugene Ng Ah Sio — trader/analyst, sporadic but specific. { "handle": "eugene", "display_name": "Eugene Ng Ah Sio", "feed_url": "https://eugene.substack.com/feed", }, # ── DeFi journalism (Substack-style RSS) ───────────────────────────── # The Defiant — Camila Russo's team. DeFi-focused news with frequent # protocol + token mentions. Free RSS, ~100 entries. { "handle": "thedefiant", "display_name": "The Defiant", "feed_url": "https://www.thedefiant.io/api/feed", "source": "blog", }, # ── Major crypto podcasts (Megaphone / Simplecast RSS) ─────────────── # Show notes are 1-6K chars — long enough for AI to pull out tickers # and theses. Bootstrap is capped at max_new=20/run so a 600-episode # backlog spreads across ~30 days. # # Empire (Blockworks) — Jason Yanowitz + Santiago Roel Santos. Weekly # crypto+macro interviews. Show notes name protocols + price calls. { "handle": "empire", "display_name": "Empire Podcast (Blockworks)", "feed_url": "https://feeds.megaphone.fm/empire", "source": "podcast", }, # 0xResearch (Blockworks) — Boccaccio + Dan Smith. Protocol research # deep-dives, real revenue/usage discussion. Highest signal density of # the Blockworks shows. { "handle": "0xresearch", "display_name": "0xResearch (Blockworks)", "feed_url": "https://feeds.megaphone.fm/0xresearch", "source": "podcast", }, # Lightspeed (Blockworks) — Mert Mumtaz (Helius CEO) + Garrett Harper. # Solana ecosystem focus — SOL, JUP, JTO, PUMP, validator economics. { "handle": "lightspeed", "display_name": "Lightspeed (Solana, Blockworks)", "feed_url": "https://feeds.megaphone.fm/lightspeed", "source": "podcast", }, # Unchained — Laura Shin. Long interview format with founders and # traders. Show notes are 6K+ chars (near-transcript). { "handle": "unchained", "display_name": "Unchained (Laura Shin)", "feed_url": "https://www.unchainedcrypto.com/feed/", "source": "podcast", }, # Bankless podcast — Ryan Sean Adams + David Hoffman. ETH-focused but # covers all majors. 4K char show notes. Largest crypto-native podcast. # NOTE: previous feed `simplecast.com/MLdpYXYI` was actually Robert # Breedlove's "What is Money" show — wrong feed. libsyn is canonical. { "handle": "bankless", "display_name": "Bankless Podcast", "feed_url": "https://bankless.libsyn.com/rss", "source": "podcast", }, # Bell Curve (Multicoin) — Mike Ippolito + Jason Yanowitz + Myles Snider. # 350 episodes, weekly macro+crypto roundup. Multicoin's portfolio shows # up frequently (SOL, JTO, JUP, Helium, Render). 1.2K show notes. { "handle": "bellcurve", "display_name": "Bell Curve (Multicoin)", "feed_url": "https://feeds.megaphone.fm/bellcurve", "source": "podcast", }, # The Scoop (The Block) — Frank Chaparro interviews founders + traders. # 110 episodes, ~700 char show notes. Strong on infrastructure/exchange # deals (Hyperliquid, Coinbase, Binance dynamics). { "handle": "thescoop", "display_name": "The Scoop (The Block)", "feed_url": "https://feeds.megaphone.fm/the-scoop", "source": "podcast", }, # ── Research newsletters (long-form, high-signal) ──────────────────── # Reflexivity Research — Will Clemente + Sam Rule. On-chain BTC analysis # and macro pieces. 20 entries, 8K char essays. Concrete on-chain calls. { "handle": "reflexivity", "display_name": "Reflexivity Research (Will Clemente)", "feed_url": "https://reflexivityresearch.substack.com/feed", }, # TFTC — Marty Bent's "Bitcoin Brief" newsletter (also a podcast feed). # 11K char issues, daily Bitcoin + policy. Pure BTC focus but covers # legislation/macro that moves BTC. { "handle": "tftc", "display_name": "TFTC / Bitcoin Brief (Marty Bent)", "feed_url": "https://tftc.io/feed", "source": "blog", }, ] # Back-compat alias — older imports referenced SUBSTACK_KOLS. SUBSTACK_KOLS = KOL_FEEDS _TAG_RE = re.compile(r"<[^>]+>") _WHITESPACE_RE = re.compile(r"[ \t]+") _BLANKLINES_RE = re.compile(r"\n{3,}") def _html_to_text(html: str) -> str: """Cheap HTML → text. Good enough for Substack which uses simple markup; if we ever need real parsing, swap to bs4 (not currently a dep).""" # Newlines for block-level closes so paragraphs survive s = re.sub(r"", "\n", html, flags=re.I) s = re.sub(r"", "\n", s, flags=re.I) s = _TAG_RE.sub("", s) # HTML entities — feedparser usually decodes these but be safe s = (s.replace("&", "&").replace("<", "<").replace(">", ">") .replace(""", '"').replace("’", "'").replace("“", '"') .replace("”", '"').replace(" ", " ")) s = _WHITESPACE_RE.sub(" ", s) s = _BLANKLINES_RE.sub("\n\n", s) return s.strip() def _entry_body(entry) -> str: """Pull the richest body field available from a feedparser entry.""" if entry.get("content"): # content is a list of {value, type} return entry["content"][0].get("value", "") or "" return entry.get("summary") or entry.get("description") or "" def _parse_pub(entry) -> Optional[datetime]: raw = entry.get("published") or entry.get("updated") if not raw: return None try: dt = parsedate_to_datetime(raw) # Always normalize to naive UTC. Previously used .astimezone() which # converts to *local* time → 8-hour skew when server runs in CST. # Affects: divergence window matching, digest 'since' filter, UI display. if dt.tzinfo: dt = dt.astimezone(timezone.utc).replace(tzinfo=None) return dt except Exception: return None async def _fetch_feed(feed_url: str) -> list: """feedparser is sync; do the HTTP fetch through httpx for timeout control + uniformity with the rest of the codebase, then hand bytes to feedparser.""" async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client: r = await client.get(feed_url, headers={"User-Agent": "TrumpSignal/1.0 KOL-tracker"}) r.raise_for_status() parsed = feedparser.parse(r.content) return list(parsed.entries or []) async def _ingest_kol( session: AsyncSession, kol: dict, *, analyze: bool = True, max_new: int = 20, ) -> dict: """Ingest one KOL feed. max_new caps first-run cost for high-volume feeds (e.g. Delphi podcast has 478 episodes). Subsequent runs only see truly new entries so the cap rarely triggers after bootstrap.""" handle = kol["handle"] feed_url = kol["feed_url"] src = kol.get("source") or "substack" # substack | podcast | blog stats = {"handle": handle, "source": src, "new": 0, "skipped": 0, "analyzed": 0, "errors": 0} try: entries = await _fetch_feed(feed_url) except Exception as e: logger.warning("[kol_substack] fetch failed for %s: %s", handle, e) stats["errors"] += 1 return stats for entry in entries: if stats["new"] >= max_new: logger.info("[kol_substack] %s hit max_new=%d cap; rest deferred to next run", handle, max_new) break # Podcast feeds (Buzzsprout, etc.) have no ; use enclosure URL or entry id. url = entry.get("link") if not url: enclosures = entry.get("enclosures") or [] if enclosures: url = enclosures[0].get("href") if not url: url = entry.get("id") # e.g. "Buzzsprout-19123172" if not url: continue # Dedupe by (source, external_id=url). We also check against the # legacy "substack" source so podcast/blog re-tags don't double-insert # entries the old code already wrote. existing = await session.execute( select(KolPost).where( KolPost.source.in_([src, "substack"]), KolPost.external_id == url, ) ) row = existing.scalar_one_or_none() if row is not None: stats["skipped"] += 1 continue html = _entry_body(entry) text = _html_to_text(html) if not text: continue pub = _parse_pub(entry) or utcnow() title = entry.get("title") or None body_hash = hashlib.sha256(text.encode("utf-8")).hexdigest() row = KolPost( kol_handle=handle, source=src, external_id=url, url=url, title=title, published_at=pub, raw_text=text, content_hash=body_hash, ) session.add(row) await session.flush() # get id for logging stats["new"] += 1 logger.info("[kol_substack] new post %s id=%s title=%r", handle, row.id, title) if analyze: try: result = await kol_analysis.extract_kol_signal( handle=handle, source=src, title=title, body=text, ) if result.get("error"): stats["errors"] += 1 else: import json as _json row.summary = result.get("summary") row.tickers_json = _json.dumps(result.get("tickers") or [], ensure_ascii=False) row.analyzed_at = utcnow() row.analysis_model = result.get("model") row.analysis_version = result.get("version") stats["analyzed"] += 1 except Exception as e: logger.warning("[kol_substack] analysis failed for %s post %s: %s", handle, row.id, e) stats["errors"] += 1 await session.commit() return stats async def run_substack_poll(*, analyze: bool = True) -> list[dict]: """Poll every configured KOL feed once. Despite the legacy name this now covers Substack essays, Medium blogs, and major crypto podcasts via RSS. Returns per-KOL stats.""" results = [] async with AsyncSessionLocal() as session: for kol in KOL_FEEDS: stats = await _ingest_kol(session, kol, analyze=analyze) results.append(stats) logger.info("[kol_substack] poll done: %s", results) return results