k
d6c802ef26
Batch of the pre-launch audit campaign (BUG-01…14 plus three new features): Pricing / TP-SL protection - Add app/services/hl_price_feed.py: supplemental HL allMids poller for HL-native assets (HYPE, PURR) not listed on Binance. Pumps price_store + tp_sl_monitor.on_price_tick so bot trades on these assets keep full stop-loss / take-profit / trailing protection instead of max-hold only. - Wire feed into main.py lifespan (startup task + graceful shutdown cancel). Telegram - Add format_trump_mention + PATH B in _dispatch: crypto-relevant Trump posts with no directional signal (relevant=True, signal=hold) now alert the public channel only (no per-subscriber noise). - Rate limiter (slowapi) on the API; assorted bot/digest fixes. KOL on-chain - seed_kol_wallets.py: KOL_FEEDS coverage cross-check; reversibly deactivate orphaned wallets (handle not in KOL_FEEDS → can never produce divergence) so the scanner stops burning cycles on them. Tests / misc - Fix brittle test_macro_ahr999_uses_same_formula_as_scanner: mock now uses realistic ms timestamps so the in-progress-day drop fires, matching the fetcher's bar count (was 0.3179 vs 0.3178 off-by-one). - Refresh stale notify_signal comment in truth_social.py. Frontend reduce-action type fix lives in the sibling repo. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
463 lines
19 KiB
Python
463 lines
19 KiB
Python
"""KOL Substack RSS ingester.
|
|
|
|
Polls each tracked KOL's Substack feed, dedupes by URL, stores raw post,
|
|
then hands off to kol_analysis.extract_kol_signal and writes the result
|
|
back onto the same row.
|
|
|
|
Substack RSS embeds the full post HTML in <content:encoded>. We strip HTML
|
|
to plain text before storage + analysis. Hayes posts are typically 50K+
|
|
chars of body — the extractor truncates internally.
|
|
|
|
Daily cadence is plenty (Hayes posts ~monthly, Substack updates feed within
|
|
minutes of publish). Call run_substack_poll() from the APScheduler.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from email.utils import parsedate_to_datetime
|
|
from typing import Iterable, Optional
|
|
|
|
import feedparser
|
|
import httpx
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import KolPost, utcnow
|
|
from app.services import kol_analysis
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Curated B-tier KOL feeds. Handle is the canonical key. `source` is the
|
|
# DB column ("substack" | "podcast" | "blog"); empty defaults to "substack"
|
|
# for legacy entries. Twitter-only KOLs come in a separate ingester.
|
|
#
|
|
# When adding a new feed:
|
|
# 1. curl + grep '<item>' to confirm it returns entries.
|
|
# 2. Inspect entry summary/content length — AI extraction needs ≥300 chars
|
|
# of body per post or it just hallucinates a topic line. (Headlines-only
|
|
# feeds like Vitalik's blog need a follow-up HTML fetch, deferred.)
|
|
# 3. Add with a sensible handle + display_name.
|
|
KOL_FEEDS: list[dict] = [
|
|
# ── Substack essayists (long-form thesis pieces) ─────────────────────
|
|
{
|
|
"handle": "cryptohayes",
|
|
"display_name": "Arthur Hayes",
|
|
"feed_url": "https://cryptohayes.substack.com/feed",
|
|
},
|
|
# Placeholder VC (Joel Monegro / Chris Burniske). Token-focused VC, posts
|
|
# long-form thesis pieces every 1-3 months that map directly to their
|
|
# portfolio bets (Solana staking, L1 monetary premium, etc.).
|
|
{
|
|
"handle": "placeholder",
|
|
"display_name": "Placeholder VC",
|
|
"feed_url": "https://www.placeholder.vc/blog?format=rss",
|
|
},
|
|
# Dragonfly Capital research blog on Medium — free, active (10+ posts).
|
|
# dragonfly.xyz/blog/rss.xml returns 0 (paywall). medium.com/dragonfly-research
|
|
# is the team's public research arm: airdrops, DeFi, protocol deep-dives.
|
|
{
|
|
"handle": "dragonfly",
|
|
"display_name": "Dragonfly Capital",
|
|
"feed_url": "https://medium.com/feed/dragonfly-research",
|
|
},
|
|
# Andy Constan's Substack is paywalled (RSS returns 0). Keeping for any
|
|
# occasional public teaser. Forward Guidance podcast (Blockworks) features
|
|
# him weekly but is macro/equities-focused — not crypto-coin-specific enough
|
|
# to extract ticker signals from episode descriptions.
|
|
{
|
|
"handle": "dampedspring",
|
|
"display_name": "Damped Spring / Andy Constan",
|
|
"feed_url": "https://dampedspring.substack.com/feed",
|
|
},
|
|
# Nic Carter's Substack is paywalled (RSS returns 0). His Medium feed is
|
|
# FREE and active — different URL, same author, real content.
|
|
{
|
|
"handle": "niccarter",
|
|
"display_name": "Nic Carter (Castle Island)",
|
|
"feed_url": "https://medium.com/feed/@nic__carter",
|
|
},
|
|
# Delphi Digital podcast (Buzzsprout) — 478 episodes, active May 2025.
|
|
# Public, free. Episode descriptions name specific protocols / tokens with
|
|
# thesis framing — good extraction signal. delphidigital.io/feed returns 0.
|
|
{
|
|
"handle": "delphi",
|
|
"display_name": "Delphi Digital (Podcast)",
|
|
"feed_url": "https://rss.buzzsprout.com/2609274.rss",
|
|
},
|
|
# ── Newly added (verified live + active) ─────────────────────────────
|
|
# Anthony Pompliano — Pomp Investments. Active monthly+ on macro/crypto.
|
|
{
|
|
"handle": "pomp",
|
|
"display_name": "Anthony Pompliano (Pomp Letter)",
|
|
"feed_url": "https://pomp.substack.com/feed",
|
|
},
|
|
# The DeFi Edge — researcher who writes 1-2 deep dives per month on
|
|
# tokens / sectors. Real thesis + position-aware framing.
|
|
{
|
|
"handle": "thedefiedge",
|
|
"display_name": "The DeFi Edge",
|
|
"feed_url": "https://thedefiedge.com/feed/",
|
|
},
|
|
# Eugene Ng Ah Sio — trader/analyst, sporadic but specific.
|
|
{
|
|
"handle": "eugene",
|
|
"display_name": "Eugene Ng Ah Sio",
|
|
"feed_url": "https://eugene.substack.com/feed",
|
|
},
|
|
# ── DeFi journalism (Substack-style RSS) ─────────────────────────────
|
|
# The Defiant — Camila Russo's team. DeFi-focused news with frequent
|
|
# protocol + token mentions. Free RSS, ~100 entries.
|
|
{
|
|
"handle": "thedefiant",
|
|
"display_name": "The Defiant",
|
|
"feed_url": "https://www.thedefiant.io/api/feed",
|
|
"source": "blog",
|
|
},
|
|
# ── Major crypto podcasts (Megaphone / Simplecast RSS) ───────────────
|
|
# Show notes are 1-6K chars — long enough for AI to pull out tickers
|
|
# and theses. Bootstrap is capped at max_new=20/run so a 600-episode
|
|
# backlog spreads across ~30 days.
|
|
#
|
|
# Empire (Blockworks) — Jason Yanowitz + Santiago Roel Santos. Weekly
|
|
# crypto+macro interviews. Show notes name protocols + price calls.
|
|
{
|
|
"handle": "empire",
|
|
"display_name": "Empire Podcast (Blockworks)",
|
|
"feed_url": "https://feeds.megaphone.fm/empire",
|
|
"source": "podcast",
|
|
},
|
|
# 0xResearch (Blockworks) — Boccaccio + Dan Smith. Protocol research
|
|
# deep-dives, real revenue/usage discussion. Highest signal density of
|
|
# the Blockworks shows.
|
|
{
|
|
"handle": "0xresearch",
|
|
"display_name": "0xResearch (Blockworks)",
|
|
"feed_url": "https://feeds.megaphone.fm/0xresearch",
|
|
"source": "podcast",
|
|
},
|
|
# Lightspeed (Blockworks) — Mert Mumtaz (Helius CEO) + Garrett Harper.
|
|
# Solana ecosystem focus — SOL, JUP, JTO, PUMP, validator economics.
|
|
{
|
|
"handle": "lightspeed",
|
|
"display_name": "Lightspeed (Solana, Blockworks)",
|
|
"feed_url": "https://feeds.megaphone.fm/lightspeed",
|
|
"source": "podcast",
|
|
},
|
|
# Unchained — Laura Shin. Long interview format with founders and
|
|
# traders. Show notes are 6K+ chars (near-transcript).
|
|
{
|
|
"handle": "unchained",
|
|
"display_name": "Unchained (Laura Shin)",
|
|
"feed_url": "https://www.unchainedcrypto.com/feed/",
|
|
"source": "podcast",
|
|
},
|
|
# Bankless podcast — Ryan Sean Adams + David Hoffman. ETH-focused but
|
|
# covers all majors. 4K char show notes. Largest crypto-native podcast.
|
|
# NOTE: previous feed `simplecast.com/MLdpYXYI` was actually Robert
|
|
# Breedlove's "What is Money" show — wrong feed. libsyn is canonical.
|
|
{
|
|
"handle": "bankless",
|
|
"display_name": "Bankless Podcast",
|
|
"feed_url": "https://bankless.libsyn.com/rss",
|
|
"source": "podcast",
|
|
},
|
|
# Bell Curve (Multicoin) — Mike Ippolito + Jason Yanowitz + Myles Snider.
|
|
# 350 episodes, weekly macro+crypto roundup. Multicoin's portfolio shows
|
|
# up frequently (SOL, JTO, JUP, Helium, Render). 1.2K show notes.
|
|
{
|
|
"handle": "bellcurve",
|
|
"display_name": "Bell Curve (Multicoin)",
|
|
"feed_url": "https://feeds.megaphone.fm/bellcurve",
|
|
"source": "podcast",
|
|
},
|
|
# The Scoop (The Block) — Frank Chaparro interviews founders + traders.
|
|
# 110 episodes, ~700 char show notes. Strong on infrastructure/exchange
|
|
# deals (Hyperliquid, Coinbase, Binance dynamics).
|
|
{
|
|
"handle": "thescoop",
|
|
"display_name": "The Scoop (The Block)",
|
|
"feed_url": "https://feeds.megaphone.fm/the-scoop",
|
|
"source": "podcast",
|
|
},
|
|
# ── Research newsletters (long-form, high-signal) ────────────────────
|
|
# Reflexivity Research — Will Clemente + Sam Rule. On-chain BTC analysis
|
|
# and macro pieces. 20 entries, 8K char essays. Concrete on-chain calls.
|
|
{
|
|
"handle": "reflexivity",
|
|
"display_name": "Reflexivity Research (Will Clemente)",
|
|
"feed_url": "https://reflexivityresearch.substack.com/feed",
|
|
},
|
|
# TFTC — Marty Bent's "Bitcoin Brief" newsletter (also a podcast feed).
|
|
# 11K char issues, daily Bitcoin + policy. Pure BTC focus but covers
|
|
# legislation/macro that moves BTC.
|
|
{
|
|
"handle": "tftc",
|
|
"display_name": "TFTC / Bitcoin Brief (Marty Bent)",
|
|
"feed_url": "https://tftc.io/feed",
|
|
"source": "blog",
|
|
},
|
|
# ── Macro research & on-chain analysis (deep, independent thinkers) ───
|
|
# Checkmate (James Check) — ex-Glassnode lead analyst. Combines on-chain
|
|
# data with macro (bonds, energy, commodities). Very active (2-3x/week).
|
|
# Concrete BTC cycle calls with high conviction.
|
|
{
|
|
"handle": "checkmate",
|
|
"display_name": "Checkmate (James Check)",
|
|
"feed_url": "https://newsletter.checkonchain.com/feed",
|
|
"source": "blog",
|
|
},
|
|
# CoinMetrics "State of the Network" — weekly data-driven research.
|
|
# Covers specific protocols/tokens with on-chain metrics. Stable Tuesday
|
|
# cadence, institutional quality.
|
|
{
|
|
"handle": "coinmetrics",
|
|
"display_name": "CoinMetrics (State of the Network)",
|
|
"feed_url": "https://coinmetrics.substack.com/feed",
|
|
},
|
|
# Willy Woo — one of the most recognized BTC on-chain analysts. Models
|
|
# like NVT, difficulty ribbon. Clear directional calls on BTC cycle phase.
|
|
{
|
|
"handle": "willywoo",
|
|
"display_name": "Willy Woo (Bitcoin Vector)",
|
|
"feed_url": "https://willywoo.substack.com/feed",
|
|
},
|
|
# Glassnode Insights — institutional on-chain research. Monthly deep
|
|
# dives on holder behavior, market structure, accumulation/distribution.
|
|
{
|
|
"handle": "glassnode",
|
|
"display_name": "Glassnode Research",
|
|
"feed_url": "https://insights.glassnode.com/rss/",
|
|
"source": "blog",
|
|
},
|
|
# Lyn Alden — independent macro analyst. Analyzes BTC through the lens of
|
|
# global liquidity, fiscal dominance, sovereign debt. Infrequent (monthly)
|
|
# but extremely high quality. Mostly BTC-only signals.
|
|
{
|
|
"handle": "lynalden",
|
|
"display_name": "Lyn Alden",
|
|
"feed_url": "https://www.lynalden.com/feed/",
|
|
"source": "blog",
|
|
},
|
|
# Bitcoin Magazine — high-frequency news + institutional adoption analysis.
|
|
# AI will extract tickers from policy/corporate BTC reserve articles.
|
|
{
|
|
"handle": "bitcoinmag",
|
|
"display_name": "Bitcoin Magazine",
|
|
"feed_url": "https://bitcoinmagazine.com/feed",
|
|
"source": "blog",
|
|
},
|
|
# ── Derivatives positioning & institutional weeklies (2026-05-29) ────
|
|
# Deribit Insights / Block Scholes — weekly options & derivatives data
|
|
# (gamma exposure, IV skew, BTC/ETH positioning). Fills the derivatives
|
|
# coverage gap — no other feed in this list tracks options flow.
|
|
{
|
|
"handle": "deribit",
|
|
"display_name": "Deribit Insights (Block Scholes)",
|
|
"feed_url": "https://insights.deribit.com/feed/",
|
|
"source": "blog",
|
|
},
|
|
# Bitfinex Alpha — weekly BTC structure analysis (leverage, ETF flows,
|
|
# on-chain profit-taking). Institutional voice, weekly cadence, concrete
|
|
# directional framing. Complements Reflexivity's monthly cadence.
|
|
{
|
|
"handle": "bitfinex",
|
|
"display_name": "Bitfinex Alpha",
|
|
"feed_url": "https://blog.bitfinex.com/feed/",
|
|
"source": "blog",
|
|
},
|
|
# Forward Guidance (Blockworks) — Felix Jauvin macro-interview podcast.
|
|
# Fed policy, AI/liquidity, rates ↔ crypto. Same Blockworks quality bar
|
|
# as Empire / Bell Curve / 0xResearch. Replaces dampedspring as the
|
|
# primary macro signal source (her substack is paywalled).
|
|
{
|
|
"handle": "forwardguidance",
|
|
"display_name": "Forward Guidance (Blockworks)",
|
|
"feed_url": "https://feeds.megaphone.fm/forwardguidance",
|
|
"source": "podcast",
|
|
},
|
|
]
|
|
|
|
# Back-compat alias — older imports referenced SUBSTACK_KOLS.
|
|
SUBSTACK_KOLS = KOL_FEEDS
|
|
|
|
|
|
_TAG_RE = re.compile(r"<[^>]+>")
|
|
_WHITESPACE_RE = re.compile(r"[ \t]+")
|
|
_BLANKLINES_RE = re.compile(r"\n{3,}")
|
|
|
|
|
|
def _html_to_text(html: str) -> str:
|
|
"""Cheap HTML → text. Good enough for Substack which uses simple markup;
|
|
if we ever need real parsing, swap to bs4 (not currently a dep)."""
|
|
# Newlines for block-level closes so paragraphs survive
|
|
s = re.sub(r"</(p|div|h[1-6]|li|br)\s*>", "\n", html, flags=re.I)
|
|
s = re.sub(r"<br\s*/?>", "\n", s, flags=re.I)
|
|
s = _TAG_RE.sub("", s)
|
|
# HTML entities — feedparser usually decodes these but be safe
|
|
s = (s.replace("&", "&").replace("<", "<").replace(">", ">")
|
|
.replace(""", '"').replace("’", "'").replace("“", '"')
|
|
.replace("”", '"').replace(" ", " "))
|
|
s = _WHITESPACE_RE.sub(" ", s)
|
|
s = _BLANKLINES_RE.sub("\n\n", s)
|
|
return s.strip()
|
|
|
|
|
|
def _entry_body(entry) -> str:
|
|
"""Pull the richest body field available from a feedparser entry."""
|
|
if entry.get("content"):
|
|
# content is a list of {value, type}
|
|
return entry["content"][0].get("value", "") or ""
|
|
return entry.get("summary") or entry.get("description") or ""
|
|
|
|
|
|
def _parse_pub(entry) -> Optional[datetime]:
|
|
raw = entry.get("published") or entry.get("updated")
|
|
if not raw:
|
|
return None
|
|
try:
|
|
dt = parsedate_to_datetime(raw)
|
|
# Always normalize to naive UTC. Previously used .astimezone() which
|
|
# converts to *local* time → 8-hour skew when server runs in CST.
|
|
# Affects: divergence window matching, digest 'since' filter, UI display.
|
|
if dt.tzinfo:
|
|
dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
|
|
return dt
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
async def _fetch_feed(feed_url: str) -> list:
|
|
"""feedparser is sync; do the HTTP fetch through httpx for timeout
|
|
control + uniformity with the rest of the codebase, then hand bytes
|
|
to feedparser."""
|
|
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
|
|
r = await client.get(feed_url, headers={"User-Agent": "TrumpSignal/1.0 KOL-tracker"})
|
|
r.raise_for_status()
|
|
parsed = feedparser.parse(r.content)
|
|
return list(parsed.entries or [])
|
|
|
|
|
|
async def _ingest_kol(
|
|
session: AsyncSession,
|
|
kol: dict,
|
|
*,
|
|
analyze: bool = True,
|
|
max_new: int = 20,
|
|
) -> dict:
|
|
"""Ingest one KOL feed. max_new caps first-run cost for high-volume feeds
|
|
(e.g. Delphi podcast has 478 episodes). Subsequent runs only see truly new
|
|
entries so the cap rarely triggers after bootstrap."""
|
|
handle = kol["handle"]
|
|
feed_url = kol["feed_url"]
|
|
src = kol.get("source") or "substack" # substack | podcast | blog
|
|
stats = {"handle": handle, "source": src,
|
|
"new": 0, "skipped": 0, "analyzed": 0, "errors": 0}
|
|
|
|
try:
|
|
entries = await _fetch_feed(feed_url)
|
|
except Exception as e:
|
|
logger.warning("[kol_substack] fetch failed for %s: %s", handle, e)
|
|
stats["errors"] += 1
|
|
return stats
|
|
|
|
for entry in entries:
|
|
if stats["new"] >= max_new:
|
|
logger.info("[kol_substack] %s hit max_new=%d cap; rest deferred to next run",
|
|
handle, max_new)
|
|
break
|
|
|
|
# Podcast feeds (Buzzsprout, etc.) have no <link>; use enclosure URL or entry id.
|
|
url = entry.get("link")
|
|
if not url:
|
|
enclosures = entry.get("enclosures") or []
|
|
if enclosures:
|
|
url = enclosures[0].get("href")
|
|
if not url:
|
|
url = entry.get("id") # e.g. "Buzzsprout-19123172"
|
|
if not url:
|
|
continue
|
|
|
|
# Dedupe by (source, external_id=url). We also check against the
|
|
# legacy "substack" source so podcast/blog re-tags don't double-insert
|
|
# entries the old code already wrote.
|
|
existing = await session.execute(
|
|
select(KolPost).where(
|
|
KolPost.source.in_([src, "substack"]),
|
|
KolPost.external_id == url,
|
|
)
|
|
)
|
|
row = existing.scalar_one_or_none()
|
|
if row is not None:
|
|
stats["skipped"] += 1
|
|
continue
|
|
|
|
html = _entry_body(entry)
|
|
text = _html_to_text(html)
|
|
if not text:
|
|
continue
|
|
|
|
pub = _parse_pub(entry) or utcnow()
|
|
title = entry.get("title") or None
|
|
body_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
row = KolPost(
|
|
kol_handle=handle,
|
|
source=src,
|
|
external_id=url,
|
|
url=url,
|
|
title=title,
|
|
published_at=pub,
|
|
raw_text=text,
|
|
content_hash=body_hash,
|
|
)
|
|
session.add(row)
|
|
await session.flush() # get id for logging
|
|
stats["new"] += 1
|
|
logger.info("[kol_substack] new post %s id=%s title=%r", handle, row.id, title)
|
|
|
|
if analyze:
|
|
try:
|
|
result = await kol_analysis.extract_kol_signal(
|
|
handle=handle,
|
|
source=src,
|
|
title=title,
|
|
body=text,
|
|
)
|
|
if result.get("error"):
|
|
stats["errors"] += 1
|
|
else:
|
|
import json as _json
|
|
row.summary = result.get("summary")
|
|
row.tickers_json = _json.dumps(result.get("tickers") or [],
|
|
ensure_ascii=False)
|
|
row.analyzed_at = utcnow()
|
|
row.analysis_model = result.get("model")
|
|
row.analysis_version = result.get("version")
|
|
stats["analyzed"] += 1
|
|
except Exception as e:
|
|
logger.warning("[kol_substack] analysis failed for %s post %s: %s",
|
|
handle, row.id, e)
|
|
stats["errors"] += 1
|
|
|
|
await session.commit()
|
|
return stats
|
|
|
|
|
|
async def run_substack_poll(*, analyze: bool = True) -> list[dict]:
|
|
"""Poll every configured KOL feed once. Despite the legacy name this now
|
|
covers Substack essays, Medium blogs, and major crypto podcasts via RSS.
|
|
Returns per-KOL stats."""
|
|
results = []
|
|
async with AsyncSessionLocal() as session:
|
|
for kol in KOL_FEEDS:
|
|
stats = await _ingest_kol(session, kol, analyze=analyze)
|
|
results.append(stats)
|
|
logger.info("[kol_substack] poll done: %s", results)
|
|
return results
|