k
54884f3e24
Feed-health pass over KOL_FEEDS: - raoulpal: stale Substack (last 2024-05) → Real Vision podcast feed - dampedspring: paywalled (0 entries) → free "Damped Spring 101" Substack - unchained: Cloudflare 403 → canonical Megaphone podcast feed - lynalden: Cloudflare 202 → FeedBurner mirror - glassnode: recovered via httpx http2=True (was 403 on HTTP/1.1) - browser User-Agent + Accept headers on feed fetch - removed dead feeds with no active replacement: placeholder, dragonfly, niccarter, eugene - pin h2==4.3.0 (required by http2=True) All 25 remaining feeds verified fetching real body content; newest post per feed ≤88d. Bundles in-flight KOL-module work already in the working tree (kol_x ingest, migration 027, tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
522 lines
23 KiB
Python
522 lines
23 KiB
Python
"""
|
||
X (Twitter) post semantic analysis — KOL real-time signals.
|
||
|
||
X posts are fundamentally different from Substack long-form articles:
|
||
- 280 chars: conclusion IS the content, no room for hedging
|
||
- Latency matters: "just bought SOL" is a minutes-level signal
|
||
- Noise ratio is extreme: 95%+ of posts are irrelevant
|
||
- Retweet/quote patterns must be detected and handled differently
|
||
- Position statements ("added", "trimmed", "exit") are high-value
|
||
- Thread context matters: a post may continue a thought from above
|
||
|
||
Design philosophy:
|
||
- Strict NOISE default (opposite of buy/sell). Most X posts should
|
||
be filtered out. The cost of false-positive on X is high because
|
||
KOLs tweet constantly.
|
||
- Three tiers of output:
|
||
TRADE_SIGNAL → actionable now (position statement + specific asset)
|
||
DIRECTIONAL → clear view, no explicit position action
|
||
NOISE → everything else (no tickers, no stance, filler)
|
||
- talks_vs_trades_flag: X posts often reveal real positions
|
||
("taking profits" while publicly bullish = divergence)
|
||
|
||
Output feeds into kol_divergence.py for cross-referencing with on-chain.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
from datetime import datetime, timezone
|
||
from typing import Optional
|
||
|
||
from openai import AsyncOpenAI
|
||
from app.config import settings
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
ANALYSIS_VERSION = "x-v2" # v2: tone-is-not-content rule + meme calibration example
|
||
ANTHROPIC_MODEL = "claude-haiku-4-5-20251001"
|
||
|
||
_anthropic_client = None
|
||
_openai_client: Optional[AsyncOpenAI] = None
|
||
|
||
|
||
def _use_anthropic() -> bool:
|
||
return bool(settings.anthropic_api_key)
|
||
|
||
|
||
def _anth():
|
||
global _anthropic_client
|
||
if _anthropic_client is None:
|
||
import anthropic as _a
|
||
_anthropic_client = _a.AsyncAnthropic(api_key=settings.anthropic_api_key)
|
||
return _anthropic_client
|
||
|
||
|
||
def _oai() -> AsyncOpenAI:
|
||
global _openai_client
|
||
if _openai_client is None:
|
||
_openai_client = AsyncOpenAI(
|
||
api_key=settings.ai_api_key,
|
||
base_url=settings.ai_base_url,
|
||
)
|
||
return _openai_client
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────
|
||
# SYSTEM PROMPT
|
||
# ─────────────────────────────────────────────────────────────────────
|
||
SYSTEM_PROMPT = """\
|
||
You are a signal extraction system for a crypto trading platform. You read
|
||
X (Twitter) posts from known crypto KOLs and decide whether they contain
|
||
a tradeable signal.
|
||
|
||
═══════════════════════════════════════════════════════════════════════
|
||
NOISE IS THE DEFAULT — internalize this
|
||
═══════════════════════════════════════════════════════════════════════
|
||
KOLs post 20-50 times per day. Most of it is:
|
||
- gm / wagmi / vibes / lifestyle
|
||
- macro commentary with no specific call
|
||
- reactions to news without a position
|
||
- vague encouragement ("BTC is king")
|
||
- promotional content / sponsored
|
||
- replies to followers (context-dependent, usually not standalone)
|
||
- jokes, memes, personal life
|
||
|
||
ALL of the above → tier: "noise", tickers: []
|
||
|
||
Only extract signals when the post contains CLEAR, EXPLICIT information
|
||
about what the KOL is doing or believes RIGHT NOW. Ambiguity → noise.
|
||
|
||
TONE IS NOT CONTENT. A joke, meme, celebratory, or emoji-spam tone
|
||
("Arise Chikun!", "Yachtzee", "Meow", "😘😘😘", "WAGMI lfg") does NOT make a
|
||
post noise when it still carries an EXPLICIT ticker + direction. Score the
|
||
claim, ignore the wrapper. "$WLD initiate bull market 😘😘😘" is a directional
|
||
bullish call on WLD — NOT noise. Only drop to noise when the ticker or the
|
||
direction is genuinely absent/vague, never merely because the wording is silly.
|
||
|
||
═══════════════════════════════════════════════════════════════════════
|
||
THREE SIGNAL TIERS
|
||
═══════════════════════════════════════════════════════════════════════
|
||
|
||
TIER 1 — TRADE_SIGNAL (rarest, highest value)
|
||
The KOL states an ACTUAL POSITION ACTION in this post:
|
||
"just bought", "added more", "long from here", "trimming my",
|
||
"sold my", "exit", "took profits", "cutting", "building position",
|
||
"DCA'd", "我建仓了", "已经上车", "减仓了", "清仓了"
|
||
Must reference a SPECIFIC asset (not just "the market").
|
||
Conviction ≥ 0.7 required for TRADE_SIGNAL.
|
||
|
||
TIER 2 — DIRECTIONAL (moderate value)
|
||
The KOL expresses a SPECIFIC, CURRENT directional view on a named asset:
|
||
"SOL looking strong here", "ETH is going to $5k", "BTC at support",
|
||
"I think [TICKER] breaks down", "watching [TICKER] for entry"
|
||
No explicit position action, but a clear current opinion.
|
||
Conviction 0.4–0.7 typical.
|
||
|
||
TIER 3 — NOISE (default)
|
||
Everything else. When in doubt, NOISE.
|
||
|
||
═══════════════════════════════════════════════════════════════════════
|
||
POST TYPE DETECTION — apply before scoring
|
||
═══════════════════════════════════════════════════════════════════════
|
||
Detect what kind of post this is before scoring content:
|
||
|
||
"original" — KOL's own thought, highest signal value
|
||
"reply" — reply to another user (context missing → usually noise
|
||
unless the post is fully self-contained)
|
||
"retweet" — RT of someone else's post without commentary → NOISE
|
||
"quote" — RT with commentary → score the commentary only,
|
||
ignore the original post content
|
||
"thread_cont"— continues a thread (may lack context → lower conviction)
|
||
|
||
Retweets without commentary are ALWAYS noise regardless of content.
|
||
|
||
═══════════════════════════════════════════════════════════════════════
|
||
TALKS VS TRADES FLAG — the platform's core value
|
||
═══════════════════════════════════════════════════════════════════════
|
||
Set talks_vs_trades_flag=true when you detect a MISMATCH between:
|
||
- What the KOL says publicly (bullish/bearish narrative)
|
||
- What they reveal about their ACTUAL position in the same post
|
||
|
||
Classic divergence patterns in X posts:
|
||
1. "BTC is going to $200k 🚀" + "took some chips off the table"
|
||
→ Narrative: ultra bullish. Action: selling. FLAG.
|
||
2. "This market is trash, crypto is dead" + "accumulated more $ETH"
|
||
→ Narrative: bearish. Action: buying. FLAG.
|
||
3. "Stay strong, don't sell!" + reveals they have <5% allocation
|
||
→ Narrative: encouraging others to hold. Position: minimal. FLAG.
|
||
4. "Not financial advice but..." followed by extreme conviction call
|
||
→ Common hedge used right before a large directional call. Reduce
|
||
conviction by 0.1. Not a flag by itself.
|
||
5. "I was wrong about X, but now I think Y"
|
||
→ Explicit reversal/stance_change. Mark stance_change=true.
|
||
|
||
═══════════════════════════════════════════════════════════════════════
|
||
CRYPTO SLANG DECODER — handle these correctly
|
||
═══════════════════════════════════════════════════════════════════════
|
||
"aping in" / "aped" → bought (often large, impulsive)
|
||
"degen'd" → took a risky position
|
||
"paper hands" → sold too early (about someone else, not actionable)
|
||
"diamond hands" → holding through drawdown (not a new position)
|
||
"ngmi" → bearish on a token or person
|
||
"wagmi" → general optimism, NOT a signal
|
||
"rekt" → lost money, position closed
|
||
"accumulated" / "acc" → bought (ongoing or past)
|
||
"trimmed" / "trim" → partially sold
|
||
"bags" / "holding bags"→ holding a position
|
||
"flipped" → changed direction (stance_change=true)
|
||
"this is the one" / "THE entry" → high conviction call
|
||
"under the radar" → bullish on an obscure asset
|
||
"cook" → doing something well, bullish signal about a project
|
||
"cooked" (about a token) → bearish, likely dying/failed
|
||
|
||
═══════════════════════════════════════════════════════════════════════
|
||
EXAMPLES — calibrate against these
|
||
═══════════════════════════════════════════════════════════════════════
|
||
|
||
[TRADE_SIGNAL — fire]
|
||
|
||
POST: "Aped into $SOL at $145. Full size. We go."
|
||
→ tier: "trade_signal", action: "buy", asset: SOL, conviction: 0.92,
|
||
timeframe: "immediate", talks_vs_trades_flag: false
|
||
|
||
POST: "Trimming half my $ETH here. Not selling the thesis but locking
|
||
profits at 3x. Will re-add lower."
|
||
→ tier: "trade_signal", action: "reduce", asset: ETH, conviction: 0.85,
|
||
timeframe: "immediate", talks_vs_trades_flag: false
|
||
|
||
POST: "This is cooked. Sold everything. Moving to stable until macro clears."
|
||
→ tier: "trade_signal", action: "sell", asset: null (multiple/all),
|
||
conviction: 0.88, timeframe: "immediate"
|
||
|
||
[DIRECTIONAL — extract but lower weight]
|
||
|
||
POST: "SOL is setting up for a breakout. $180 is the line to watch."
|
||
→ tier: "directional", action: "bullish", asset: SOL, conviction: 0.58,
|
||
timeframe: "days"
|
||
|
||
POST: "ETH dominance will crush alts this cycle. The flippening is real."
|
||
→ tier: "directional", action: "bullish", asset: ETH, conviction: 0.52,
|
||
timeframe: "months"
|
||
|
||
POST: "Arise Chikun! $WLD initiate bull market. Yachtzee 😘😘😘😘"
|
||
→ tier: "directional", action: "bullish", asset: WLD, conviction: 0.7,
|
||
timeframe: "immediate" (meme/celebration tone does NOT override the
|
||
explicit "$WLD initiate bull market" call — score the claim, not the vibe)
|
||
|
||
[NOISE — output noise, empty tickers]
|
||
|
||
POST: "gm everyone 🌅"
|
||
→ tier: "noise", tickers: []
|
||
|
||
POST: "The market is wild lmao. Stay safe everyone."
|
||
→ tier: "noise", tickers: []
|
||
|
||
POST: "Interesting take by @SomeAnalyst on the Fed" [with RT of article]
|
||
→ tier: "noise" (retweet of someone else, no original content)
|
||
|
||
POST: "Not financial advice but BTC is really interesting here 👀"
|
||
→ tier: "noise" (too vague, "interesting" is not a stance)
|
||
|
||
POST: "Just attended [Conference]. Amazing speakers. Building in crypto
|
||
is incredible."
|
||
→ tier: "noise" (no market signal)
|
||
|
||
═══════════════════════════════════════════════════════════════════════
|
||
OUTPUT FORMAT (strict JSON, no markdown)
|
||
═══════════════════════════════════════════════════════════════════════
|
||
{
|
||
"post_type": "original" | "reply" | "retweet" | "quote" | "thread_cont",
|
||
"tier": "trade_signal" | "directional" | "noise",
|
||
"summary": "<one sentence in ENGLISH, ≤80 chars. State what the KOL
|
||
is saying/doing. 'Noise: gm post' if tier is noise.>",
|
||
"tickers": [
|
||
{
|
||
"ticker": "<UPPERCASE symbol>",
|
||
"action": "buy" | "sell" | "reduce" | "bullish" | "bearish" | "mention",
|
||
"conviction": <float 0.0-1.0>,
|
||
"timeframe": "immediate" | "hours" | "days" | "weeks" | "months" | "unspecified",
|
||
"stance_change": <true if this reverses a previously stated position in this post>,
|
||
"quote": "<verbatim phrase from the post supporting this, ≤100 chars>"
|
||
}
|
||
],
|
||
"talks_vs_trades_flag": <true | false>,
|
||
"has_price_target": <true if a specific price level is mentioned>,
|
||
"price_targets": [
|
||
{ "ticker": "SOL", "price": 180, "direction": "up" | "down" }
|
||
],
|
||
"sentiment": "bullish" | "bearish" | "neutral",
|
||
"reasoning": "<one sentence: why this tier? What specific phrase drove the decision?>"
|
||
}
|
||
|
||
HARD RULES:
|
||
• tier == "noise" → tickers MUST be [], talks_vs_trades_flag MUST be false
|
||
• tier == "trade_signal" → at least one ticker with action in {buy, sell, reduce}
|
||
AND conviction ≥ 0.7
|
||
• post_type == "retweet" (RT without commentary) → tier MUST be "noise"
|
||
• If no explicit price target → price_targets: []
|
||
• conviction < 0.4 → tier cannot be "trade_signal"
|
||
"""
|
||
|
||
# ─────────────────────────────────────────────────────────────────────
|
||
# USER PROMPT TEMPLATE
|
||
# ─────────────────────────────────────────────────────────────────────
|
||
USER_TEMPLATE = """\
|
||
KOL handle: @{handle}
|
||
Follower tier: {follower_tier}
|
||
Post time (UTC): {posted_at}
|
||
Current UTC hour: {hour} ({liquidity_note})
|
||
|
||
POST:
|
||
\"\"\"{text}\"\"\"
|
||
|
||
{thread_context}
|
||
Score this post. Default to NOISE unless there is explicit, specific signal."""
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────
|
||
# Helpers
|
||
# ─────────────────────────────────────────────────────────────────────
|
||
def _liquidity_note(hour: int) -> str:
|
||
if 3 <= hour < 9:
|
||
return "Asia overnight — thin liquidity, be extra strict"
|
||
if 13 <= hour < 21:
|
||
return "US session — normal strictness"
|
||
return "Off-peak hours"
|
||
|
||
|
||
def _follower_tier(follower_count: Optional[int]) -> str:
|
||
if follower_count is None:
|
||
return "unknown"
|
||
if follower_count >= 500_000:
|
||
return "mega (500k+)"
|
||
if follower_count >= 100_000:
|
||
return "large (100k-500k)"
|
||
if follower_count >= 20_000:
|
||
return "mid (20k-100k)"
|
||
return "small (<20k)"
|
||
|
||
|
||
_FALLBACK = {
|
||
"post_type": "original",
|
||
"tier": "noise",
|
||
"summary": None,
|
||
"tickers": [],
|
||
"talks_vs_trades_flag": False,
|
||
"has_price_target": False,
|
||
"price_targets": [],
|
||
"sentiment": "neutral",
|
||
"reasoning": "",
|
||
"model": None,
|
||
"version": ANALYSIS_VERSION,
|
||
"error": None,
|
||
}
|
||
|
||
VALID_ACTIONS = {"buy", "sell", "reduce", "bullish", "bearish", "mention"}
|
||
VALID_TIMEFRAMES = {"immediate", "hours", "days", "weeks", "months", "unspecified"}
|
||
VALID_TIERS = {"trade_signal", "directional", "noise"}
|
||
VALID_POST_TYPES = {"original", "reply", "retweet", "quote", "thread_cont"}
|
||
|
||
|
||
async def analyze_x_post(
|
||
*,
|
||
handle: str,
|
||
text: str,
|
||
posted_at: Optional[str] = None,
|
||
follower_count: Optional[int] = None,
|
||
thread_context: Optional[str] = None,
|
||
model: Optional[str] = None,
|
||
) -> dict:
|
||
"""Score an X post from a crypto KOL.
|
||
|
||
Args:
|
||
handle: Twitter/X handle (without @)
|
||
text: Post text (already stripped of URLs if desired)
|
||
posted_at: ISO UTC timestamp of the post
|
||
follower_count: Follower count for context (affects tier label)
|
||
thread_context: Optional: preceding posts in thread, ≤500 chars
|
||
model: Override model. Defaults to ai_model (quality path).
|
||
|
||
Returns dict with: post_type, tier, summary, tickers, talks_vs_trades_flag,
|
||
has_price_target, price_targets, sentiment, reasoning, model, version.
|
||
Errors return a noise-safe fallback (never raises).
|
||
"""
|
||
stripped = (text or "").strip()
|
||
if not stripped:
|
||
out = dict(_FALLBACK)
|
||
out["error"] = "empty post"
|
||
return out
|
||
|
||
# Fast pre-filter: bare RT with no added text is always noise
|
||
low = stripped.lower()
|
||
if low.startswith("rt @") and len(stripped) < 300:
|
||
out = dict(_FALLBACK)
|
||
out["post_type"] = "retweet"
|
||
out["reasoning"] = "Pre-filtered: bare retweet with no commentary."
|
||
return out
|
||
|
||
hour = datetime.now(timezone.utc).hour
|
||
thread_block = (
|
||
f"Thread context (preceding posts):\n\"\"\"\n{thread_context[:500]}\n\"\"\""
|
||
if thread_context else ""
|
||
)
|
||
|
||
user_prompt = USER_TEMPLATE.format(
|
||
handle=handle,
|
||
follower_tier=_follower_tier(follower_count),
|
||
posted_at=posted_at or "unknown",
|
||
hour=hour,
|
||
liquidity_note=_liquidity_note(hour),
|
||
text=stripped[:1000],
|
||
thread_context=thread_block,
|
||
)
|
||
|
||
use_anth = _use_anthropic()
|
||
if model is None:
|
||
# X analysis is near-real-time but less latency-critical than Trump.
|
||
# Use the quality model for better signal/noise discrimination.
|
||
model = ANTHROPIC_MODEL if use_anth else settings.ai_model
|
||
|
||
try:
|
||
if use_anth:
|
||
msg = await _anth().messages.create(
|
||
model=model,
|
||
max_tokens=800,
|
||
temperature=0.1,
|
||
system=SYSTEM_PROMPT,
|
||
messages=[{"role": "user", "content": user_prompt}],
|
||
)
|
||
raw = (msg.content[0].text if msg.content else "").strip()
|
||
else:
|
||
is_reasoning = any(x in model for x in ("pro", "reasoner", "r1", "think"))
|
||
kwargs: dict = {
|
||
"model": model,
|
||
"messages": [
|
||
{"role": "system", "content": SYSTEM_PROMPT},
|
||
{"role": "user", "content": user_prompt},
|
||
],
|
||
"max_tokens": 2000 if is_reasoning else 800,
|
||
}
|
||
if not is_reasoning:
|
||
kwargs["temperature"] = 0.1
|
||
kwargs["response_format"] = {"type": "json_object"}
|
||
resp = await _oai().chat.completions.create(**kwargs)
|
||
raw = (resp.choices[0].message.content or "").strip()
|
||
|
||
# Strip fences
|
||
if raw.startswith("```"):
|
||
lines = raw.split("\n")
|
||
raw = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
|
||
|
||
result = json.loads(raw)
|
||
|
||
except json.JSONDecodeError as exc:
|
||
logger.error("x_analysis JSON parse error for @%s: %s", handle, exc)
|
||
out = dict(_FALLBACK)
|
||
out["error"] = f"parse_error: {exc}"
|
||
return out
|
||
except Exception as exc:
|
||
logger.error("x_analysis API error for @%s: %s", handle, exc)
|
||
out = dict(_FALLBACK)
|
||
out["error"] = f"api_error: {exc}"
|
||
return out
|
||
|
||
# ── Normalize ────────────────────────────────────────────────────
|
||
post_type = (result.get("post_type") or "original").lower()
|
||
if post_type not in VALID_POST_TYPES:
|
||
post_type = "original"
|
||
|
||
tier = (result.get("tier") or "noise").lower()
|
||
if tier not in VALID_TIERS:
|
||
tier = "noise"
|
||
|
||
# Enforce: retweet → noise
|
||
if post_type == "retweet":
|
||
tier = "noise"
|
||
|
||
# Normalize tickers
|
||
raw_tickers = result.get("tickers") or []
|
||
cleaned_tickers = []
|
||
if tier != "noise":
|
||
for t in raw_tickers:
|
||
if not isinstance(t, dict):
|
||
continue
|
||
sym = (t.get("ticker") or "").strip().upper()
|
||
if not sym or len(sym) > 12:
|
||
continue
|
||
action = (t.get("action") or "mention").lower()
|
||
if action not in VALID_ACTIONS:
|
||
action = "mention"
|
||
try:
|
||
conv = float(t.get("conviction") or 0)
|
||
except (TypeError, ValueError):
|
||
conv = 0.0
|
||
conv = max(0.0, min(1.0, conv))
|
||
timeframe = (t.get("timeframe") or "unspecified").lower()
|
||
if timeframe not in VALID_TIMEFRAMES:
|
||
timeframe = "unspecified"
|
||
cleaned_tickers.append({
|
||
"ticker": sym,
|
||
"action": action,
|
||
"conviction": round(conv, 2),
|
||
"timeframe": timeframe,
|
||
"stance_change": bool(t.get("stance_change", False)),
|
||
"quote": (t.get("quote") or "")[:100],
|
||
})
|
||
|
||
# Enforce: trade_signal requires ≥1 ticker with buy/sell/reduce + conviction ≥ 0.7
|
||
if tier == "trade_signal":
|
||
strong = [
|
||
t for t in cleaned_tickers
|
||
if t["action"] in {"buy", "sell", "reduce"} and t["conviction"] >= 0.7
|
||
]
|
||
if not strong:
|
||
tier = "directional" # downgrade rather than drop
|
||
|
||
# Enforce: noise → empty tickers
|
||
if tier == "noise":
|
||
cleaned_tickers = []
|
||
|
||
sentiment = (result.get("sentiment") or "neutral").lower()
|
||
if sentiment not in ("bullish", "bearish", "neutral"):
|
||
sentiment = "neutral"
|
||
|
||
talks_vs_trades = bool(result.get("talks_vs_trades_flag", False))
|
||
if tier == "noise":
|
||
talks_vs_trades = False
|
||
|
||
# Price targets
|
||
has_price_target = bool(result.get("has_price_target", False))
|
||
raw_pts = result.get("price_targets") or []
|
||
price_targets = []
|
||
for pt in raw_pts:
|
||
if not isinstance(pt, dict):
|
||
continue
|
||
ticker = (pt.get("ticker") or "").upper()
|
||
try:
|
||
price = float(pt.get("price") or 0)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
direction = (pt.get("direction") or "up").lower()
|
||
if ticker and price > 0 and direction in ("up", "down"):
|
||
price_targets.append({"ticker": ticker, "price": price, "direction": direction})
|
||
if not price_targets:
|
||
has_price_target = False
|
||
|
||
return {
|
||
"post_type": post_type,
|
||
"tier": tier,
|
||
"summary": (result.get("summary") or "").strip() or None,
|
||
"tickers": cleaned_tickers,
|
||
"talks_vs_trades_flag": talks_vs_trades,
|
||
"has_price_target": has_price_target,
|
||
"price_targets": price_targets,
|
||
"sentiment": sentiment,
|
||
"reasoning": str(result.get("reasoning", ""))[:500],
|
||
"model": model,
|
||
"version": ANALYSIS_VERSION,
|
||
"error": None,
|
||
}
|