"""KOL Substack RSS ingester. Polls each tracked KOL's Substack feed, dedupes by URL, stores raw post, then hands off to kol_analysis.extract_kol_signal and writes the result back onto the same row. Substack RSS embeds the full post HTML in . We strip HTML to plain text before storage + analysis. Hayes posts are typically 50K+ chars of body — the extractor truncates internally. Daily cadence is plenty (Hayes posts ~monthly, Substack updates feed within minutes of publish). Call run_substack_poll() from the APScheduler. """ from __future__ import annotations import hashlib import logging import re from datetime import datetime, timezone from email.utils import parsedate_to_datetime from typing import Iterable, Optional import feedparser import httpx from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.database import AsyncSessionLocal from app.models import KolPost, utcnow from app.services import kol_analysis logger = logging.getLogger(__name__) # Curated B-tier KOL feeds. Handle is the canonical key. `source` is the # DB column ("substack" | "podcast" | "blog"); empty defaults to "substack" # for legacy entries. Twitter-only KOLs come in a separate ingester. # # When adding a new feed: # 1. curl + grep '' to confirm it returns entries. # 2. Inspect entry summary/content length — AI extraction needs ≥300 chars # of body per post or it just hallucinates a topic line. (Headlines-only # feeds like Vitalik's blog need a follow-up HTML fetch, deferred.) # 3. Add with a sensible handle + display_name. # 4. ⚠️ The KOL feed COUNT is hardcoded in the FRONTEND for SEO/marketing # (it can't read this list cross-repo). If len(KOL_FEEDS) changes, update # every "N KOL feeds" mention in the frontend repo: # app/layout.tsx (JSON-LD ×2), app/page.tsx (×3 incl. metric), # app/[locale]/kol/page.tsx (×4), app/[locale]/kol/KolPageClient.tsx # (subtitle "and N more" = count-3), app/[locale]/glossary/page.tsx, # public/llms.txt + llms-full.txt, app/opengraph-image.tsx. # Currently len(KOL_FEEDS) == 25. (X-only KOLs live in kol_x.X_KOLS.) # (2026-06-09: dropped from 29 → 25; removed placeholder, dragonfly, # niccarter, eugene as dead feeds. Frontend count mentions need updating.) KOL_FEEDS: list[dict] = [ # ── Substack essayists (long-form thesis pieces) ───────────────────── { "handle": "cryptohayes", "display_name": "Arthur Hayes", "feed_url": "https://cryptohayes.substack.com/feed", }, # Raoul Pal — Real Vision / Global Macro Investor founder. "Short Excerpts # Raoul Pal — his public Substack (raoulpal.substack.com/feed) went STALE # (last post 2024-05). Replaced 2026-06-09 with the Real Vision official # podcast feed (feeds.megaphone.fm/realvision) — daily, free, 2000+ eps, # full episode descriptions with macro/crypto thesis. It's the whole Real # Vision channel (not Raoul-only), but high signal + fresh. Same canonical # handle as his X (@RaoulGMI in kol_x) so long-form + real-time aggregate. { "handle": "raoulpal", "display_name": "Raoul Pal (Real Vision)", "feed_url": "https://feeds.megaphone.fm/realvision", "source": "podcast", }, # REMOVED 2026-06-09 — dead feeds, no active replacement exists: # • placeholder (Placeholder VC) — last post 2025-09, blog is their only # public source, no newsletter. ~266d stale. # • dragonfly (Dragonfly Research, Medium) — last post 2025-03 (~454d). # dragonfly.xyz has no working RSS (JS-rendered, all endpoints empty); # Haseeb's medium/@hosseeb is even older (2024). # If either resumes regular publishing with a real RSS, re-add here. # Andy Constan — his old Substack (dampedspring.substack.com) returned 0 # entries (paywalled). Replaced 2026-06-09 with his FREE long-form Substack # "Damped Spring 101" (dampedspring101.substack.com), which he's publicly # committed to keeping free — active, real macro essays (liquidity, rates, # positioning). His deepest paid research stays gated, but this carries # genuine thesis content suitable for ticker/direction extraction. { "handle": "dampedspring", "display_name": "Damped Spring / Andy Constan", "feed_url": "https://dampedspring101.substack.com/feed", }, # REMOVED 2026-06-09 — niccarter (Nic Carter / Castle Island). Substack # paywalled (0 entries); Medium feed went stale (last 2025-07, ~316d). # No active free replacement. Re-add if he resumes a public RSS. # Delphi Digital podcast (Buzzsprout) — 478 episodes, active May 2025. # Public, free. Episode descriptions name specific protocols / tokens with # thesis framing — good extraction signal. delphidigital.io/feed returns 0. { "handle": "delphi", "display_name": "Delphi Digital (Podcast)", "feed_url": "https://rss.buzzsprout.com/2609274.rss", }, # ── Newly added (verified live + active) ───────────────────────────── # Anthony Pompliano — Pomp Investments. Active monthly+ on macro/crypto. { "handle": "pomp", "display_name": "Anthony Pompliano (Pomp Letter)", "feed_url": "https://pomp.substack.com/feed", }, # The DeFi Edge — researcher who writes 1-2 deep dives per month on # tokens / sectors. Real thesis + position-aware framing. { "handle": "thedefiedge", "display_name": "The DeFi Edge", "feed_url": "https://thedefiedge.com/feed/", }, # REMOVED 2026-06-09 — eugene (Eugene Ng Ah Sio). Substack effectively # dormant: only 5 entries, last 2025-05 (~399d). No alternative source. # ── DeFi journalism (Substack-style RSS) ───────────────────────────── # The Defiant — Camila Russo's team. DeFi-focused news with frequent # protocol + token mentions. Free RSS, ~100 entries. { "handle": "thedefiant", "display_name": "The Defiant", "feed_url": "https://www.thedefiant.io/api/feed", "source": "blog", }, # ── Major crypto podcasts (Megaphone / Simplecast RSS) ─────────────── # Show notes are 1-6K chars — long enough for AI to pull out tickers # and theses. Bootstrap is capped at max_new=20/run so a 600-episode # backlog spreads across ~30 days. # # Empire (Blockworks) — Jason Yanowitz + Santiago Roel Santos. Weekly # crypto+macro interviews. Show notes name protocols + price calls. { "handle": "empire", "display_name": "Empire Podcast (Blockworks)", "feed_url": "https://feeds.megaphone.fm/empire", "source": "podcast", }, # 0xResearch (Blockworks) — Boccaccio + Dan Smith. Protocol research # deep-dives, real revenue/usage discussion. Highest signal density of # the Blockworks shows. { "handle": "0xresearch", "display_name": "0xResearch (Blockworks)", "feed_url": "https://feeds.megaphone.fm/0xresearch", "source": "podcast", }, # Lightspeed (Blockworks) — Mert Mumtaz (Helius CEO) + Garrett Harper. # Solana ecosystem focus — SOL, JUP, JTO, PUMP, validator economics. { "handle": "lightspeed", "display_name": "Lightspeed (Solana, Blockworks)", "feed_url": "https://feeds.megaphone.fm/lightspeed", "source": "podcast", }, # Unchained — Laura Shin. Long interview format with founders and traders. # The website RSS (unchainedcrypto.com/feed) is Cloudflare-blocked (403, # even HTTP/2 + browser UA). Switched 2026-06-09 to the canonical Megaphone # podcast feed (resolved via iTunes lookup id=1123922160) — 1100+ episodes, # active daily, 1.5-2K char show notes naming protocols/tokens. { "handle": "unchained", "display_name": "Unchained (Laura Shin)", "feed_url": "https://feeds.megaphone.fm/LSHML4761942757", "source": "podcast", }, # Bankless podcast — Ryan Sean Adams + David Hoffman. ETH-focused but # covers all majors. 4K char show notes. Largest crypto-native podcast. # NOTE: previous feed `simplecast.com/MLdpYXYI` was actually Robert # Breedlove's "What is Money" show — wrong feed. libsyn is canonical. { "handle": "bankless", "display_name": "Bankless Podcast", "feed_url": "https://bankless.libsyn.com/rss", "source": "podcast", }, # Bell Curve (Multicoin) — Mike Ippolito + Jason Yanowitz + Myles Snider. # 350 episodes, weekly macro+crypto roundup. Multicoin's portfolio shows # up frequently (SOL, JTO, JUP, Helium, Render). 1.2K show notes. { "handle": "bellcurve", "display_name": "Bell Curve (Multicoin)", "feed_url": "https://feeds.megaphone.fm/bellcurve", "source": "podcast", }, # The Scoop (The Block) — Frank Chaparro interviews founders + traders. # 110 episodes, ~700 char show notes. Strong on infrastructure/exchange # deals (Hyperliquid, Coinbase, Binance dynamics). { "handle": "thescoop", "display_name": "The Scoop (The Block)", "feed_url": "https://feeds.megaphone.fm/the-scoop", "source": "podcast", }, # ── Research newsletters (long-form, high-signal) ──────────────────── # Reflexivity Research — Will Clemente + Sam Rule. On-chain BTC analysis # and macro pieces. 20 entries, 8K char essays. Concrete on-chain calls. { "handle": "reflexivity", "display_name": "Reflexivity Research (Will Clemente)", "feed_url": "https://reflexivityresearch.substack.com/feed", }, # TFTC — Marty Bent's "Bitcoin Brief" newsletter (also a podcast feed). # 11K char issues, daily Bitcoin + policy. Pure BTC focus but covers # legislation/macro that moves BTC. { "handle": "tftc", "display_name": "TFTC / Bitcoin Brief (Marty Bent)", "feed_url": "https://tftc.io/feed", "source": "blog", }, # ── Macro research & on-chain analysis (deep, independent thinkers) ─── # Checkmate (James Check) — ex-Glassnode lead analyst. Combines on-chain # data with macro (bonds, energy, commodities). Very active (2-3x/week). # Concrete BTC cycle calls with high conviction. { "handle": "checkmate", "display_name": "Checkmate (James Check)", "feed_url": "https://newsletter.checkonchain.com/feed", "source": "blog", }, # CoinMetrics "State of the Network" — weekly data-driven research. # Covers specific protocols/tokens with on-chain metrics. Stable Tuesday # cadence, institutional quality. { "handle": "coinmetrics", "display_name": "CoinMetrics (State of the Network)", "feed_url": "https://coinmetrics.substack.com/feed", }, # Willy Woo — one of the most recognized BTC on-chain analysts. Models # like NVT, difficulty ribbon. Clear directional calls on BTC cycle phase. { "handle": "willywoo", "display_name": "Willy Woo (Bitcoin Vector)", "feed_url": "https://willywoo.substack.com/feed", }, # Glassnode Insights — institutional on-chain research. Monthly deep # dives on holder behavior, market structure, accumulation/distribution. { "handle": "glassnode", "display_name": "Glassnode Research", "feed_url": "https://insights.glassnode.com/rss/", "source": "blog", }, # Lyn Alden — independent macro analyst. Analyzes BTC through the lens of # global liquidity, fiscal dominance, sovereign debt. Infrequent (monthly) # but extremely high quality. Mostly BTC-only signals. { "handle": "lynalden", # lynalden.com/feed is Cloudflare-blocked (202, no body). Switched # 2026-06-09 to her FeedBurner mirror, which serves fine. Note: bodies # are article EXCERPTS (~400 chars) not full text, but enough for the # AI to extract the BTC/macro thesis + direction. "display_name": "Lyn Alden", "feed_url": "https://feeds.feedburner.com/lynalden", "source": "blog", }, # Bitcoin Magazine — high-frequency news + institutional adoption analysis. # AI will extract tickers from policy/corporate BTC reserve articles. { "handle": "bitcoinmag", "display_name": "Bitcoin Magazine", "feed_url": "https://bitcoinmagazine.com/feed", "source": "blog", }, # ── Derivatives positioning & institutional weeklies (2026-05-29) ──── # Deribit Insights / Block Scholes — weekly options & derivatives data # (gamma exposure, IV skew, BTC/ETH positioning). Fills the derivatives # coverage gap — no other feed in this list tracks options flow. { "handle": "deribit", "display_name": "Deribit Insights (Block Scholes)", "feed_url": "https://insights.deribit.com/feed/", "source": "blog", }, # Bitfinex Alpha — weekly BTC structure analysis (leverage, ETF flows, # on-chain profit-taking). Institutional voice, weekly cadence, concrete # directional framing. Complements Reflexivity's monthly cadence. { "handle": "bitfinex", "display_name": "Bitfinex Alpha", "feed_url": "https://blog.bitfinex.com/feed/", "source": "blog", }, # Forward Guidance (Blockworks) — Felix Jauvin macro-interview podcast. # Fed policy, AI/liquidity, rates ↔ crypto. Same Blockworks quality bar # as Empire / Bell Curve / 0xResearch. Replaces dampedspring as the # primary macro signal source (her substack is paywalled). { "handle": "forwardguidance", "display_name": "Forward Guidance (Blockworks)", "feed_url": "https://feeds.megaphone.fm/forwardguidance", "source": "podcast", }, ] # Back-compat alias — older imports referenced SUBSTACK_KOLS. SUBSTACK_KOLS = KOL_FEEDS _TAG_RE = re.compile(r"<[^>]+>") _WHITESPACE_RE = re.compile(r"[ \t]+") _BLANKLINES_RE = re.compile(r"\n{3,}") def _html_to_text(html: str) -> str: """Cheap HTML → text. Good enough for Substack which uses simple markup; if we ever need real parsing, swap to bs4 (not currently a dep).""" # Newlines for block-level closes so paragraphs survive s = re.sub(r"", "\n", html, flags=re.I) s = re.sub(r"", "\n", s, flags=re.I) s = _TAG_RE.sub("", s) # HTML entities — feedparser usually decodes these but be safe s = (s.replace("&", "&").replace("<", "<").replace(">", ">") .replace(""", '"').replace("’", "'").replace("“", '"') .replace("”", '"').replace(" ", " ")) s = _WHITESPACE_RE.sub(" ", s) s = _BLANKLINES_RE.sub("\n\n", s) return s.strip() def _entry_body(entry) -> str: """Pull the richest body field available from a feedparser entry.""" if entry.get("content"): # content is a list of {value, type} return entry["content"][0].get("value", "") or "" return entry.get("summary") or entry.get("description") or "" def _parse_pub(entry) -> Optional[datetime]: raw = entry.get("published") or entry.get("updated") if not raw: return None try: dt = parsedate_to_datetime(raw) # Always normalize to naive UTC. Previously used .astimezone() which # converts to *local* time → 8-hour skew when server runs in CST. # Affects: divergence window matching, digest 'since' filter, UI display. if dt.tzinfo: dt = dt.astimezone(timezone.utc).replace(tzinfo=None) return dt except Exception: return None async def _fetch_feed(feed_url: str) -> list: """feedparser is sync; do the HTTP fetch through httpx for timeout control + uniformity with the rest of the codebase, then hand bytes to feedparser.""" # http2=True matters: some CDN-fronted feeds (e.g. Glassnode) return 403 # to HTTP/1.1 requests but serve fine over HTTP/2 (curl defaults to h2, # which is why they worked manually but not here). Requires the `h2` pkg. async with httpx.AsyncClient(timeout=20.0, follow_redirects=True, http2=True) as client: # Use a real browser UA. Several feeds (Glassnode, others behind a CDN) # return 403/202 to non-browser agents. A plain bot UA was silently # losing those feeds. This recovers them without per-feed special-casing. r = await client.get(feed_url, headers={ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/124.0 Safari/537.36", "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*", }) r.raise_for_status() parsed = feedparser.parse(r.content) return list(parsed.entries or []) async def _ingest_kol( session: AsyncSession, kol: dict, *, analyze: bool = True, max_new: int = 20, ) -> dict: """Ingest one KOL feed. max_new caps first-run cost for high-volume feeds (e.g. Delphi podcast has 478 episodes). Subsequent runs only see truly new entries so the cap rarely triggers after bootstrap.""" handle = kol["handle"] feed_url = kol["feed_url"] src = kol.get("source") or "substack" # substack | podcast | blog stats = {"handle": handle, "source": src, "new": 0, "skipped": 0, "analyzed": 0, "errors": 0} try: entries = await _fetch_feed(feed_url) except Exception as e: logger.warning("[kol_substack] fetch failed for %s: %s", handle, e) stats["errors"] += 1 return stats for entry in entries: if stats["new"] >= max_new: logger.info("[kol_substack] %s hit max_new=%d cap; rest deferred to next run", handle, max_new) break # Podcast feeds (Buzzsprout, etc.) have no ; use enclosure URL or entry id. url = entry.get("link") if not url: enclosures = entry.get("enclosures") or [] if enclosures: url = enclosures[0].get("href") if not url: url = entry.get("id") # e.g. "Buzzsprout-19123172" if not url: continue # Dedupe by (source, external_id=url). We also check against the # legacy "substack" source so podcast/blog re-tags don't double-insert # entries the old code already wrote. existing = await session.execute( select(KolPost).where( KolPost.source.in_([src, "substack"]), KolPost.external_id == url, ) ) row = existing.scalar_one_or_none() if row is not None: stats["skipped"] += 1 continue html = _entry_body(entry) text = _html_to_text(html) if not text: continue pub = _parse_pub(entry) or utcnow() title = entry.get("title") or None body_hash = hashlib.sha256(text.encode("utf-8")).hexdigest() row = KolPost( kol_handle=handle, source=src, external_id=url, url=url, title=title, published_at=pub, raw_text=text, content_hash=body_hash, ) session.add(row) await session.flush() # get id for logging stats["new"] += 1 logger.info("[kol_substack] new post %s id=%s title=%r", handle, row.id, title) if analyze: try: result = await kol_analysis.extract_kol_signal( handle=handle, source=src, title=title, body=text, ) if result.get("error"): stats["errors"] += 1 else: import json as _json row.summary = result.get("summary") row.tickers_json = _json.dumps(result.get("tickers") or [], ensure_ascii=False) row.analyzed_at = utcnow() row.analysis_model = result.get("model") row.analysis_version = result.get("version") # Extended analysis fields (migration 027) row.post_type = result.get("post_type") row.talks_vs_trades_flag = bool(result.get("talks_vs_trades_flag", False)) # tier (trade_signal/directional/noise) so the "Signals only" # filter + SIGNAL/VIEW badges work for non-Twitter KOLs too. row.tier = result.get("tier") stats["analyzed"] += 1 except Exception as e: logger.warning("[kol_substack] analysis failed for %s post %s: %s", handle, row.id, e) stats["errors"] += 1 await session.commit() return stats async def run_substack_poll(*, analyze: bool = True) -> list[dict]: """Poll every configured KOL feed once. Despite the legacy name this now covers Substack essays, Medium blogs, and major crypto podcasts via RSS. Returns per-KOL stats. Each KOL gets its own session so a commit failure for one does not leave a dirty session that breaks subsequent KOLs in the same run. """ results = [] for kol in KOL_FEEDS: async with AsyncSessionLocal() as session: stats = await _ingest_kol(session, kol, analyze=analyze) results.append(stats) logger.info("[kol_substack] poll done: %s", results) return results