k
54884f3e24
Feed-health pass over KOL_FEEDS: - raoulpal: stale Substack (last 2024-05) → Real Vision podcast feed - dampedspring: paywalled (0 entries) → free "Damped Spring 101" Substack - unchained: Cloudflare 403 → canonical Megaphone podcast feed - lynalden: Cloudflare 202 → FeedBurner mirror - glassnode: recovered via httpx http2=True (was 403 on HTTP/1.1) - browser User-Agent + Accept headers on feed fetch - removed dead feeds with no active replacement: placeholder, dragonfly, niccarter, eugene - pin h2==4.3.0 (required by http2=True) All 25 remaining feeds verified fetching real body content; newest post per feed ≤88d. Bundles in-flight KOL-module work already in the working tree (kol_x ingest, migration 027, tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
496 lines
21 KiB
Python
496 lines
21 KiB
Python
"""KOL Substack RSS ingester.
|
||
|
||
Polls each tracked KOL's Substack feed, dedupes by URL, stores raw post,
|
||
then hands off to kol_analysis.extract_kol_signal and writes the result
|
||
back onto the same row.
|
||
|
||
Substack RSS embeds the full post HTML in <content:encoded>. We strip HTML
|
||
to plain text before storage + analysis. Hayes posts are typically 50K+
|
||
chars of body — the extractor truncates internally.
|
||
|
||
Daily cadence is plenty (Hayes posts ~monthly, Substack updates feed within
|
||
minutes of publish). Call run_substack_poll() from the APScheduler.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import logging
|
||
import re
|
||
from datetime import datetime, timezone
|
||
from email.utils import parsedate_to_datetime
|
||
from typing import Iterable, Optional
|
||
|
||
import feedparser
|
||
import httpx
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.database import AsyncSessionLocal
|
||
from app.models import KolPost, utcnow
|
||
from app.services import kol_analysis
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# Curated B-tier KOL feeds. Handle is the canonical key. `source` is the
|
||
# DB column ("substack" | "podcast" | "blog"); empty defaults to "substack"
|
||
# for legacy entries. Twitter-only KOLs come in a separate ingester.
|
||
#
|
||
# When adding a new feed:
|
||
# 1. curl + grep '<item>' to confirm it returns entries.
|
||
# 2. Inspect entry summary/content length — AI extraction needs ≥300 chars
|
||
# of body per post or it just hallucinates a topic line. (Headlines-only
|
||
# feeds like Vitalik's blog need a follow-up HTML fetch, deferred.)
|
||
# 3. Add with a sensible handle + display_name.
|
||
# 4. ⚠️ The KOL feed COUNT is hardcoded in the FRONTEND for SEO/marketing
|
||
# (it can't read this list cross-repo). If len(KOL_FEEDS) changes, update
|
||
# every "N KOL feeds" mention in the frontend repo:
|
||
# app/layout.tsx (JSON-LD ×2), app/page.tsx (×3 incl. metric),
|
||
# app/[locale]/kol/page.tsx (×4), app/[locale]/kol/KolPageClient.tsx
|
||
# (subtitle "and N more" = count-3), app/[locale]/glossary/page.tsx,
|
||
# public/llms.txt + llms-full.txt, app/opengraph-image.tsx.
|
||
# Currently len(KOL_FEEDS) == 25. (X-only KOLs live in kol_x.X_KOLS.)
|
||
# (2026-06-09: dropped from 29 → 25; removed placeholder, dragonfly,
|
||
# niccarter, eugene as dead feeds. Frontend count mentions need updating.)
|
||
KOL_FEEDS: list[dict] = [
|
||
# ── Substack essayists (long-form thesis pieces) ─────────────────────
|
||
{
|
||
"handle": "cryptohayes",
|
||
"display_name": "Arthur Hayes",
|
||
"feed_url": "https://cryptohayes.substack.com/feed",
|
||
},
|
||
# Raoul Pal — Real Vision / Global Macro Investor founder. "Short Excerpts
|
||
# Raoul Pal — his public Substack (raoulpal.substack.com/feed) went STALE
|
||
# (last post 2024-05). Replaced 2026-06-09 with the Real Vision official
|
||
# podcast feed (feeds.megaphone.fm/realvision) — daily, free, 2000+ eps,
|
||
# full episode descriptions with macro/crypto thesis. It's the whole Real
|
||
# Vision channel (not Raoul-only), but high signal + fresh. Same canonical
|
||
# handle as his X (@RaoulGMI in kol_x) so long-form + real-time aggregate.
|
||
{
|
||
"handle": "raoulpal",
|
||
"display_name": "Raoul Pal (Real Vision)",
|
||
"feed_url": "https://feeds.megaphone.fm/realvision",
|
||
"source": "podcast",
|
||
},
|
||
# REMOVED 2026-06-09 — dead feeds, no active replacement exists:
|
||
# • placeholder (Placeholder VC) — last post 2025-09, blog is their only
|
||
# public source, no newsletter. ~266d stale.
|
||
# • dragonfly (Dragonfly Research, Medium) — last post 2025-03 (~454d).
|
||
# dragonfly.xyz has no working RSS (JS-rendered, all endpoints empty);
|
||
# Haseeb's medium/@hosseeb is even older (2024).
|
||
# If either resumes regular publishing with a real RSS, re-add here.
|
||
# Andy Constan — his old Substack (dampedspring.substack.com) returned 0
|
||
# entries (paywalled). Replaced 2026-06-09 with his FREE long-form Substack
|
||
# "Damped Spring 101" (dampedspring101.substack.com), which he's publicly
|
||
# committed to keeping free — active, real macro essays (liquidity, rates,
|
||
# positioning). His deepest paid research stays gated, but this carries
|
||
# genuine thesis content suitable for ticker/direction extraction.
|
||
{
|
||
"handle": "dampedspring",
|
||
"display_name": "Damped Spring / Andy Constan",
|
||
"feed_url": "https://dampedspring101.substack.com/feed",
|
||
},
|
||
# REMOVED 2026-06-09 — niccarter (Nic Carter / Castle Island). Substack
|
||
# paywalled (0 entries); Medium feed went stale (last 2025-07, ~316d).
|
||
# No active free replacement. Re-add if he resumes a public RSS.
|
||
# Delphi Digital podcast (Buzzsprout) — 478 episodes, active May 2025.
|
||
# Public, free. Episode descriptions name specific protocols / tokens with
|
||
# thesis framing — good extraction signal. delphidigital.io/feed returns 0.
|
||
{
|
||
"handle": "delphi",
|
||
"display_name": "Delphi Digital (Podcast)",
|
||
"feed_url": "https://rss.buzzsprout.com/2609274.rss",
|
||
},
|
||
# ── Newly added (verified live + active) ─────────────────────────────
|
||
# Anthony Pompliano — Pomp Investments. Active monthly+ on macro/crypto.
|
||
{
|
||
"handle": "pomp",
|
||
"display_name": "Anthony Pompliano (Pomp Letter)",
|
||
"feed_url": "https://pomp.substack.com/feed",
|
||
},
|
||
# The DeFi Edge — researcher who writes 1-2 deep dives per month on
|
||
# tokens / sectors. Real thesis + position-aware framing.
|
||
{
|
||
"handle": "thedefiedge",
|
||
"display_name": "The DeFi Edge",
|
||
"feed_url": "https://thedefiedge.com/feed/",
|
||
},
|
||
# REMOVED 2026-06-09 — eugene (Eugene Ng Ah Sio). Substack effectively
|
||
# dormant: only 5 entries, last 2025-05 (~399d). No alternative source.
|
||
# ── DeFi journalism (Substack-style RSS) ─────────────────────────────
|
||
# The Defiant — Camila Russo's team. DeFi-focused news with frequent
|
||
# protocol + token mentions. Free RSS, ~100 entries.
|
||
{
|
||
"handle": "thedefiant",
|
||
"display_name": "The Defiant",
|
||
"feed_url": "https://www.thedefiant.io/api/feed",
|
||
"source": "blog",
|
||
},
|
||
# ── Major crypto podcasts (Megaphone / Simplecast RSS) ───────────────
|
||
# Show notes are 1-6K chars — long enough for AI to pull out tickers
|
||
# and theses. Bootstrap is capped at max_new=20/run so a 600-episode
|
||
# backlog spreads across ~30 days.
|
||
#
|
||
# Empire (Blockworks) — Jason Yanowitz + Santiago Roel Santos. Weekly
|
||
# crypto+macro interviews. Show notes name protocols + price calls.
|
||
{
|
||
"handle": "empire",
|
||
"display_name": "Empire Podcast (Blockworks)",
|
||
"feed_url": "https://feeds.megaphone.fm/empire",
|
||
"source": "podcast",
|
||
},
|
||
# 0xResearch (Blockworks) — Boccaccio + Dan Smith. Protocol research
|
||
# deep-dives, real revenue/usage discussion. Highest signal density of
|
||
# the Blockworks shows.
|
||
{
|
||
"handle": "0xresearch",
|
||
"display_name": "0xResearch (Blockworks)",
|
||
"feed_url": "https://feeds.megaphone.fm/0xresearch",
|
||
"source": "podcast",
|
||
},
|
||
# Lightspeed (Blockworks) — Mert Mumtaz (Helius CEO) + Garrett Harper.
|
||
# Solana ecosystem focus — SOL, JUP, JTO, PUMP, validator economics.
|
||
{
|
||
"handle": "lightspeed",
|
||
"display_name": "Lightspeed (Solana, Blockworks)",
|
||
"feed_url": "https://feeds.megaphone.fm/lightspeed",
|
||
"source": "podcast",
|
||
},
|
||
# Unchained — Laura Shin. Long interview format with founders and traders.
|
||
# The website RSS (unchainedcrypto.com/feed) is Cloudflare-blocked (403,
|
||
# even HTTP/2 + browser UA). Switched 2026-06-09 to the canonical Megaphone
|
||
# podcast feed (resolved via iTunes lookup id=1123922160) — 1100+ episodes,
|
||
# active daily, 1.5-2K char show notes naming protocols/tokens.
|
||
{
|
||
"handle": "unchained",
|
||
"display_name": "Unchained (Laura Shin)",
|
||
"feed_url": "https://feeds.megaphone.fm/LSHML4761942757",
|
||
"source": "podcast",
|
||
},
|
||
# Bankless podcast — Ryan Sean Adams + David Hoffman. ETH-focused but
|
||
# covers all majors. 4K char show notes. Largest crypto-native podcast.
|
||
# NOTE: previous feed `simplecast.com/MLdpYXYI` was actually Robert
|
||
# Breedlove's "What is Money" show — wrong feed. libsyn is canonical.
|
||
{
|
||
"handle": "bankless",
|
||
"display_name": "Bankless Podcast",
|
||
"feed_url": "https://bankless.libsyn.com/rss",
|
||
"source": "podcast",
|
||
},
|
||
# Bell Curve (Multicoin) — Mike Ippolito + Jason Yanowitz + Myles Snider.
|
||
# 350 episodes, weekly macro+crypto roundup. Multicoin's portfolio shows
|
||
# up frequently (SOL, JTO, JUP, Helium, Render). 1.2K show notes.
|
||
{
|
||
"handle": "bellcurve",
|
||
"display_name": "Bell Curve (Multicoin)",
|
||
"feed_url": "https://feeds.megaphone.fm/bellcurve",
|
||
"source": "podcast",
|
||
},
|
||
# The Scoop (The Block) — Frank Chaparro interviews founders + traders.
|
||
# 110 episodes, ~700 char show notes. Strong on infrastructure/exchange
|
||
# deals (Hyperliquid, Coinbase, Binance dynamics).
|
||
{
|
||
"handle": "thescoop",
|
||
"display_name": "The Scoop (The Block)",
|
||
"feed_url": "https://feeds.megaphone.fm/the-scoop",
|
||
"source": "podcast",
|
||
},
|
||
# ── Research newsletters (long-form, high-signal) ────────────────────
|
||
# Reflexivity Research — Will Clemente + Sam Rule. On-chain BTC analysis
|
||
# and macro pieces. 20 entries, 8K char essays. Concrete on-chain calls.
|
||
{
|
||
"handle": "reflexivity",
|
||
"display_name": "Reflexivity Research (Will Clemente)",
|
||
"feed_url": "https://reflexivityresearch.substack.com/feed",
|
||
},
|
||
# TFTC — Marty Bent's "Bitcoin Brief" newsletter (also a podcast feed).
|
||
# 11K char issues, daily Bitcoin + policy. Pure BTC focus but covers
|
||
# legislation/macro that moves BTC.
|
||
{
|
||
"handle": "tftc",
|
||
"display_name": "TFTC / Bitcoin Brief (Marty Bent)",
|
||
"feed_url": "https://tftc.io/feed",
|
||
"source": "blog",
|
||
},
|
||
# ── Macro research & on-chain analysis (deep, independent thinkers) ───
|
||
# Checkmate (James Check) — ex-Glassnode lead analyst. Combines on-chain
|
||
# data with macro (bonds, energy, commodities). Very active (2-3x/week).
|
||
# Concrete BTC cycle calls with high conviction.
|
||
{
|
||
"handle": "checkmate",
|
||
"display_name": "Checkmate (James Check)",
|
||
"feed_url": "https://newsletter.checkonchain.com/feed",
|
||
"source": "blog",
|
||
},
|
||
# CoinMetrics "State of the Network" — weekly data-driven research.
|
||
# Covers specific protocols/tokens with on-chain metrics. Stable Tuesday
|
||
# cadence, institutional quality.
|
||
{
|
||
"handle": "coinmetrics",
|
||
"display_name": "CoinMetrics (State of the Network)",
|
||
"feed_url": "https://coinmetrics.substack.com/feed",
|
||
},
|
||
# Willy Woo — one of the most recognized BTC on-chain analysts. Models
|
||
# like NVT, difficulty ribbon. Clear directional calls on BTC cycle phase.
|
||
{
|
||
"handle": "willywoo",
|
||
"display_name": "Willy Woo (Bitcoin Vector)",
|
||
"feed_url": "https://willywoo.substack.com/feed",
|
||
},
|
||
# Glassnode Insights — institutional on-chain research. Monthly deep
|
||
# dives on holder behavior, market structure, accumulation/distribution.
|
||
{
|
||
"handle": "glassnode",
|
||
"display_name": "Glassnode Research",
|
||
"feed_url": "https://insights.glassnode.com/rss/",
|
||
"source": "blog",
|
||
},
|
||
# Lyn Alden — independent macro analyst. Analyzes BTC through the lens of
|
||
# global liquidity, fiscal dominance, sovereign debt. Infrequent (monthly)
|
||
# but extremely high quality. Mostly BTC-only signals.
|
||
{
|
||
"handle": "lynalden",
|
||
# lynalden.com/feed is Cloudflare-blocked (202, no body). Switched
|
||
# 2026-06-09 to her FeedBurner mirror, which serves fine. Note: bodies
|
||
# are article EXCERPTS (~400 chars) not full text, but enough for the
|
||
# AI to extract the BTC/macro thesis + direction.
|
||
"display_name": "Lyn Alden",
|
||
"feed_url": "https://feeds.feedburner.com/lynalden",
|
||
"source": "blog",
|
||
},
|
||
# Bitcoin Magazine — high-frequency news + institutional adoption analysis.
|
||
# AI will extract tickers from policy/corporate BTC reserve articles.
|
||
{
|
||
"handle": "bitcoinmag",
|
||
"display_name": "Bitcoin Magazine",
|
||
"feed_url": "https://bitcoinmagazine.com/feed",
|
||
"source": "blog",
|
||
},
|
||
# ── Derivatives positioning & institutional weeklies (2026-05-29) ────
|
||
# Deribit Insights / Block Scholes — weekly options & derivatives data
|
||
# (gamma exposure, IV skew, BTC/ETH positioning). Fills the derivatives
|
||
# coverage gap — no other feed in this list tracks options flow.
|
||
{
|
||
"handle": "deribit",
|
||
"display_name": "Deribit Insights (Block Scholes)",
|
||
"feed_url": "https://insights.deribit.com/feed/",
|
||
"source": "blog",
|
||
},
|
||
# Bitfinex Alpha — weekly BTC structure analysis (leverage, ETF flows,
|
||
# on-chain profit-taking). Institutional voice, weekly cadence, concrete
|
||
# directional framing. Complements Reflexivity's monthly cadence.
|
||
{
|
||
"handle": "bitfinex",
|
||
"display_name": "Bitfinex Alpha",
|
||
"feed_url": "https://blog.bitfinex.com/feed/",
|
||
"source": "blog",
|
||
},
|
||
# Forward Guidance (Blockworks) — Felix Jauvin macro-interview podcast.
|
||
# Fed policy, AI/liquidity, rates ↔ crypto. Same Blockworks quality bar
|
||
# as Empire / Bell Curve / 0xResearch. Replaces dampedspring as the
|
||
# primary macro signal source (her substack is paywalled).
|
||
{
|
||
"handle": "forwardguidance",
|
||
"display_name": "Forward Guidance (Blockworks)",
|
||
"feed_url": "https://feeds.megaphone.fm/forwardguidance",
|
||
"source": "podcast",
|
||
},
|
||
]
|
||
|
||
# Back-compat alias — older imports referenced SUBSTACK_KOLS.
|
||
SUBSTACK_KOLS = KOL_FEEDS
|
||
|
||
|
||
_TAG_RE = re.compile(r"<[^>]+>")
|
||
_WHITESPACE_RE = re.compile(r"[ \t]+")
|
||
_BLANKLINES_RE = re.compile(r"\n{3,}")
|
||
|
||
|
||
def _html_to_text(html: str) -> str:
|
||
"""Cheap HTML → text. Good enough for Substack which uses simple markup;
|
||
if we ever need real parsing, swap to bs4 (not currently a dep)."""
|
||
# Newlines for block-level closes so paragraphs survive
|
||
s = re.sub(r"</(p|div|h[1-6]|li|br)\s*>", "\n", html, flags=re.I)
|
||
s = re.sub(r"<br\s*/?>", "\n", s, flags=re.I)
|
||
s = _TAG_RE.sub("", s)
|
||
# HTML entities — feedparser usually decodes these but be safe
|
||
s = (s.replace("&", "&").replace("<", "<").replace(">", ">")
|
||
.replace(""", '"').replace("’", "'").replace("“", '"')
|
||
.replace("”", '"').replace(" ", " "))
|
||
s = _WHITESPACE_RE.sub(" ", s)
|
||
s = _BLANKLINES_RE.sub("\n\n", s)
|
||
return s.strip()
|
||
|
||
|
||
def _entry_body(entry) -> str:
|
||
"""Pull the richest body field available from a feedparser entry."""
|
||
if entry.get("content"):
|
||
# content is a list of {value, type}
|
||
return entry["content"][0].get("value", "") or ""
|
||
return entry.get("summary") or entry.get("description") or ""
|
||
|
||
|
||
def _parse_pub(entry) -> Optional[datetime]:
|
||
raw = entry.get("published") or entry.get("updated")
|
||
if not raw:
|
||
return None
|
||
try:
|
||
dt = parsedate_to_datetime(raw)
|
||
# Always normalize to naive UTC. Previously used .astimezone() which
|
||
# converts to *local* time → 8-hour skew when server runs in CST.
|
||
# Affects: divergence window matching, digest 'since' filter, UI display.
|
||
if dt.tzinfo:
|
||
dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
|
||
return dt
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
async def _fetch_feed(feed_url: str) -> list:
|
||
"""feedparser is sync; do the HTTP fetch through httpx for timeout
|
||
control + uniformity with the rest of the codebase, then hand bytes
|
||
to feedparser."""
|
||
# http2=True matters: some CDN-fronted feeds (e.g. Glassnode) return 403
|
||
# to HTTP/1.1 requests but serve fine over HTTP/2 (curl defaults to h2,
|
||
# which is why they worked manually but not here). Requires the `h2` pkg.
|
||
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True, http2=True) as client:
|
||
# Use a real browser UA. Several feeds (Glassnode, others behind a CDN)
|
||
# return 403/202 to non-browser agents. A plain bot UA was silently
|
||
# losing those feeds. This recovers them without per-feed special-casing.
|
||
r = await client.get(feed_url, headers={
|
||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/124.0 Safari/537.36",
|
||
"Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*",
|
||
})
|
||
r.raise_for_status()
|
||
parsed = feedparser.parse(r.content)
|
||
return list(parsed.entries or [])
|
||
|
||
|
||
async def _ingest_kol(
|
||
session: AsyncSession,
|
||
kol: dict,
|
||
*,
|
||
analyze: bool = True,
|
||
max_new: int = 20,
|
||
) -> dict:
|
||
"""Ingest one KOL feed. max_new caps first-run cost for high-volume feeds
|
||
(e.g. Delphi podcast has 478 episodes). Subsequent runs only see truly new
|
||
entries so the cap rarely triggers after bootstrap."""
|
||
handle = kol["handle"]
|
||
feed_url = kol["feed_url"]
|
||
src = kol.get("source") or "substack" # substack | podcast | blog
|
||
stats = {"handle": handle, "source": src,
|
||
"new": 0, "skipped": 0, "analyzed": 0, "errors": 0}
|
||
|
||
try:
|
||
entries = await _fetch_feed(feed_url)
|
||
except Exception as e:
|
||
logger.warning("[kol_substack] fetch failed for %s: %s", handle, e)
|
||
stats["errors"] += 1
|
||
return stats
|
||
|
||
for entry in entries:
|
||
if stats["new"] >= max_new:
|
||
logger.info("[kol_substack] %s hit max_new=%d cap; rest deferred to next run",
|
||
handle, max_new)
|
||
break
|
||
|
||
# Podcast feeds (Buzzsprout, etc.) have no <link>; use enclosure URL or entry id.
|
||
url = entry.get("link")
|
||
if not url:
|
||
enclosures = entry.get("enclosures") or []
|
||
if enclosures:
|
||
url = enclosures[0].get("href")
|
||
if not url:
|
||
url = entry.get("id") # e.g. "Buzzsprout-19123172"
|
||
if not url:
|
||
continue
|
||
|
||
# Dedupe by (source, external_id=url). We also check against the
|
||
# legacy "substack" source so podcast/blog re-tags don't double-insert
|
||
# entries the old code already wrote.
|
||
existing = await session.execute(
|
||
select(KolPost).where(
|
||
KolPost.source.in_([src, "substack"]),
|
||
KolPost.external_id == url,
|
||
)
|
||
)
|
||
row = existing.scalar_one_or_none()
|
||
if row is not None:
|
||
stats["skipped"] += 1
|
||
continue
|
||
|
||
html = _entry_body(entry)
|
||
text = _html_to_text(html)
|
||
if not text:
|
||
continue
|
||
|
||
pub = _parse_pub(entry) or utcnow()
|
||
title = entry.get("title") or None
|
||
body_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||
|
||
row = KolPost(
|
||
kol_handle=handle,
|
||
source=src,
|
||
external_id=url,
|
||
url=url,
|
||
title=title,
|
||
published_at=pub,
|
||
raw_text=text,
|
||
content_hash=body_hash,
|
||
)
|
||
session.add(row)
|
||
await session.flush() # get id for logging
|
||
stats["new"] += 1
|
||
logger.info("[kol_substack] new post %s id=%s title=%r", handle, row.id, title)
|
||
|
||
if analyze:
|
||
try:
|
||
result = await kol_analysis.extract_kol_signal(
|
||
handle=handle,
|
||
source=src,
|
||
title=title,
|
||
body=text,
|
||
)
|
||
if result.get("error"):
|
||
stats["errors"] += 1
|
||
else:
|
||
import json as _json
|
||
row.summary = result.get("summary")
|
||
row.tickers_json = _json.dumps(result.get("tickers") or [],
|
||
ensure_ascii=False)
|
||
row.analyzed_at = utcnow()
|
||
row.analysis_model = result.get("model")
|
||
row.analysis_version = result.get("version")
|
||
# Extended analysis fields (migration 027)
|
||
row.post_type = result.get("post_type")
|
||
row.talks_vs_trades_flag = bool(result.get("talks_vs_trades_flag", False))
|
||
stats["analyzed"] += 1
|
||
except Exception as e:
|
||
logger.warning("[kol_substack] analysis failed for %s post %s: %s",
|
||
handle, row.id, e)
|
||
stats["errors"] += 1
|
||
|
||
await session.commit()
|
||
return stats
|
||
|
||
|
||
async def run_substack_poll(*, analyze: bool = True) -> list[dict]:
|
||
"""Poll every configured KOL feed once. Despite the legacy name this now
|
||
covers Substack essays, Medium blogs, and major crypto podcasts via RSS.
|
||
Returns per-KOL stats.
|
||
|
||
Each KOL gets its own session so a commit failure for one does not leave
|
||
a dirty session that breaks subsequent KOLs in the same run.
|
||
"""
|
||
results = []
|
||
for kol in KOL_FEEDS:
|
||
async with AsyncSessionLocal() as session:
|
||
stats = await _ingest_kol(session, kol, analyze=analyze)
|
||
results.append(stats)
|
||
logger.info("[kol_substack] poll done: %s", results)
|
||
return results
|