trumpsignal-backend/app/services/kol_analysis.py

"""KOL post → structured signal extractor.

Takes a long-form post (Substack essay) or tweet and returns:
  - summary:  one Chinese sentence on what this post is about
  - tickers:  list of {ticker, action, conviction, quote}

action ∈ bullish | bearish | buy | sell | reduce | mention
  - buy/sell  → KOL explicitly states they bought/sold or are entering/exiting
  - reduce    → KOL is partially exiting / taking profits on an existing long
  - bullish/bearish → directional view without an explicit position statement
  - mention   → ticker appears but no clear stance (don't flood with these)

conviction ∈ 0.0–1.0
  - 0.8+  : explicit, repeated, with sizing / timing
  - 0.5–0.7 : clear view, no commitment
  - <0.5  : passing reference

Quote is the shortest verbatim sentence supporting the call.

Uses the same Anthropic client style as analysis.py. Designed to be reused
by the Trump-post AI signal module (TODO #4) — same JSON shape, just with
different KOL context strings.
"""

from __future__ import annotations

import json
import logging
from datetime import datetime, timezone
from typing import Optional

from openai import AsyncOpenAI

from app.config import settings

logger = logging.getLogger(__name__)

ANALYSIS_VERSION = "kol-v2"
ANTHROPIC_MODEL = "claude-haiku-4-5-20251001"

_anthropic_client = None
_openai_client: Optional[AsyncOpenAI] = None


def _use_anthropic() -> bool:
    return bool(settings.anthropic_api_key)


def _anth():
    global _anthropic_client
    if _anthropic_client is None:
        import anthropic as _a
        _anthropic_client = _a.AsyncAnthropic(api_key=settings.anthropic_api_key)
    return _anthropic_client


def _oai() -> AsyncOpenAI:
    global _openai_client
    if _openai_client is None:
        _openai_client = AsyncOpenAI(
            api_key=settings.ai_api_key,
            base_url=settings.ai_base_url,
        )
    return _openai_client


SYSTEM_PROMPT = """You are an analyst extracting tradeable signals from crypto KOL (Key Opinion Leader) posts.

The author is a known crypto KOL. Your job: distill what they said and which tokens they are talking about RIGHT NOW (not historical references). Pay special attention to DIVERGENCE between what they say publicly and what they imply about their actual position — this is the platform's highest-conviction signal.

Output **strict JSON only**, no markdown, no preface. Schema:

{
  "summary": "<2-3 sentences in ENGLISH, ≤200 chars total. State the author's current market thesis if they have one, or describe the post topic. Capture the key directional call if any. Always English regardless of the post's original language.>",
  "post_type": "trade_update" | "macro_thesis" | "research" | "news_recap" | "opinion" | "other",
  "tickers": [
    {
      "ticker":        "<UPPERCASE symbol, e.g. BTC, ETH, HYPE, SOL>",
      "action":        "buy" | "sell" | "bullish" | "bearish" | "reduce" | "mention",
      "conviction":    <float 0.0-1.0>,
      "timeframe":     "immediate" | "days" | "weeks" | "months" | "unspecified",
      "stance_change": <true if this contradicts or reverses what the author said in earlier parts of the same post, or walks back a previously stated position | false>,
      "quote":         "<shortest verbatim sentence from the post supporting this call, ≤200 chars. Keep original language — do not translate.>"
    }
  ],
  "talks_vs_trades_score": <float 0.0-1.0. 0 = no divergence detected. 0.5 = moderate mismatch signals. 1.0 = clear contradiction between stated narrative and implied position action. See rules below.>
}

Rules:

POST TYPE:
- "trade_update"  → author explicitly describes entering, exiting, or adjusting a position
- "macro_thesis"  → broad market view, cycle analysis, regime commentary without specific trade action
- "research"      → data-driven, analytical, with sources/charts (typical of Delphi, Glassnode, Blockworks)
- "news_recap"    → summarising recent events without strong personal view
- "opinion"       → personal take without data backing or explicit position
- "other"         → doesn't fit the above

SUMMARY:
- Always English regardless of post language.
- 2-3 sentences, ≤200 chars. Lead with the directional call if there is one.
- If no signal: describe what the post covers in plain terms.

TICKERS:
- IGNORE historical price references ("BTC bottomed at $60k earlier this year") — context, not current calls.
- IGNORE advertising/sponsor sections — cues: "sponsor", "partner", "use code", "promo code", "this episode brought to you by", "ad", "广告", "赞助", "合作方". Skip any ticker only mentioned inside such a section.
- action values:
    "buy"/"sell"   → author explicitly states a position action ("I bought", "we are long", "我们减仓了", "added to my bag", "taking profits", "已建仓", "我加仓了", "建了仓")
    "reduce"       → partially exiting or taking profits on an existing long ("trimming", "taking some off", "减了一部分", "止盈了一些")
    "bullish"      → ANY expression of upside view, including SOFT/INDIRECT/CASUAL language. Err on the side of bullish over mention.
                     English soft bullish: "X looks interesting here", "I like X at these levels", "X has room to run",
                     "wouldn't be short X", "X is setting up well", "compelling setup in X", "X is undervalued",
                     "I'd be a buyer of X", "X deserves attention here", "keeping an eye on X"
                     Chinese soft bullish: "感觉还有空间", "这里可以关注", "值得关注", "看好", "还有机会",
                     "这个位置不错", "可以考虑", "有上涨空间", "这波不错", "可以关注一下", "挺有意思的"
    "bearish"      → ANY expression of downside view or caution, including soft signals.
                     English soft bearish: "X looks extended", "I'd be careful here", "not touching X",
                     "X has further to fall", "selling into strength on X", "X is a trap"
                     Chinese soft bearish: "感觉要跌", "要小心", "不敢碰", "还会跌", "高位了", "风险很大"
    "mention"      → ticker appears but author has ZERO directional lean — pure neutral reference,
                     historical context only, or just used as an example. If there is ANY lean at all,
                     use bullish or bearish instead. Reserve mention for truly neutral uses like:
                     "BTC started in 2009" or "unlike ETH, which uses proof-of-stake"
- Dedupe per ticker — at most one entry per symbol; pick the strongest action.
- Do NOT invent tickers. Skip "$XYZ" if unsure it is a real crypto token.
- For CATEGORY-LEVEL calls (no specific ticker named), use these synthetic tickers:
    "MEME"    → author is bullish/bearish on memecoins as a category ("memes look good", "memecoin season", "梗币最近不错")
    "ALTCOIN" → author is bullish/bearish on altcoins broadly ("alts are waking up", "山寨季", "altseason")
    "DEFI"    → author is bullish/bearish on DeFi protocols broadly
    "AI"      → author is bullish/bearish on AI-related tokens broadly
  Only use synthetic tickers when the view is genuinely about the category, not just a passing mention.
- conviction: 0.8+ = explicit + repeated + sized or timed; 0.5–0.7 = clear view, no commitment; <0.5 = passing reference.
- timeframe: "immediate" = acting now or within 24h; "days" = 1–7d; "weeks" = 1–4w; "months" = 1+ months; "unspecified" = not stated.
- Do not include fiat (USD/CNY/JPY) or stablecoins (USDT/USDC/DAI/FRAX/USDE) unless the post's main thesis is about them.

TALKS-VS-TRADES SCORE (talks_vs_trades_score):
This is the platform's most important signal. Score 0.0–1.0. Raise the score when you detect narrative-position mismatches:

Score 0.7–1.0 (strong divergence):
- Author writes a bullish thesis but describes reducing, trimming, or taking profits on the same asset
- Author is publicly bearish but mentions "accumulating", "adding at these levels", "买了一些"
- Post is notably silent on an asset the author has been loudly bullish on recently (avoidance signal)
- Stated high conviction ("this is THE entry", "strongest conviction of my career") but disclosed position size is very small
- Author hedges every bullish statement with "but I could be wrong", "just my opinion", "not financial advice" applied unevenly (selective hedging)

Score 0.4–0.69 (moderate divergence):
- Author mentions "risk management" or "waiting for confirmation" alongside a bullish thesis
- Mixed signals: bullish long-term but explicitly neutral or cautious short-term on the same asset
- Language shift: previous posts were assertive, this one uses softer language without explanation

Score 0.1–0.39 (weak divergence):
- Minor hedges in an otherwise consistent narrative
- Vague "taking profits" without clear contradiction of the main thesis

Score 0.0:
- Narrative and any position signals are fully consistent
- Post has no position signals at all (pure macro commentary)

Chinese-language divergence cues to detect: "减仓了" (reduced position), "止盈" (taking profit), "降低了仓位" (lowered position), "观望" (watching/waiting), "不着急" (in no hurry), "先不动" (staying put), "谨慎" (cautious), "控制仓位" (managing position size), "风控" (risk management).
"""


USER_TEMPLATE = """Today is {today_utc}.
KOL handle: {handle}
Source: {source}
Title: {title}

Post body:
\"\"\"
{body}
\"\"\"
"""


def _truncate(text: str, max_chars: int = 24000) -> str:
    """Substack essays can be 50K+ chars. Haiku handles it but we cap to
    control cost. Keep head + tail since conclusions often appear at the end."""
    if len(text) <= max_chars:
        return text
    head = max_chars * 2 // 3
    tail = max_chars - head
    return text[:head] + "\n\n[...trimmed...]\n\n" + text[-tail:]


def _parse_json(raw: str) -> dict:
    raw = raw.strip()
    if raw.startswith("```"):
        # strip fenced code block
        raw = raw.split("\n", 1)[1] if "\n" in raw else raw
        if raw.endswith("```"):
            raw = raw.rsplit("```", 1)[0]
        raw = raw.strip()
    # Some models prepend "json" after the fence
    if raw.startswith("json"):
        raw = raw[4:].strip()
    return json.loads(raw)


async def extract_kol_signal(
    *,
    handle: str,
    source: str,
    title: Optional[str],
    body: str,
    model: Optional[str] = None,
) -> dict:
    """Run the extractor. Returns {summary, tickers, model, version}.

    Returns an empty-but-valid dict on parse/API failure rather than raising —
    the caller stores the post regardless; an unanalyzed post can be retried.
    """
    today_utc = datetime.now(timezone.utc).strftime("%Y-%m-%d")
    user = USER_TEMPLATE.format(
        today_utc=today_utc,
        handle=handle,
        source=source,
        title=title or "",
        body=_truncate(body),
    )

    use_anth = _use_anthropic()
    if model is None:
        # KOL analysis is a daily batch job, not latency-sensitive. Use the
        # higher-quality `ai_model` (DeepSeek v4 Pro / reasoning) rather than
        # the live `ai_live_model` (flash) reserved for Trump real-time path.
        model = ANTHROPIC_MODEL if use_anth else settings.ai_model

    try:
        if use_anth:
            msg = await _anth().messages.create(
                model=model,
                max_tokens=2000,   # raised: complex essays with many tickers were truncating at 1500
                temperature=0.1,
                system=SYSTEM_PROMPT,
                messages=[{"role": "user", "content": user}],
            )
            raw = (msg.content[0].text if msg.content else "").strip()
        else:
            # OpenAI-compatible (DeepSeek). Reasoning models need higher tokens
            # + no temperature; flash/chat models are fine with both.
            is_reasoning = any(x in model for x in ("pro", "reasoner", "r1", "think"))
            kwargs = {"model": model, "messages": [
                {"role": "system", "content": SYSTEM_PROMPT},
                {"role": "user", "content": user},
            ], "max_tokens": 4000 if is_reasoning else 2000}
            if not is_reasoning:
                kwargs["temperature"] = 0.1
            # JSON mode — DeepSeek + OpenAI both support response_format.
            # Eliminates fenced/preface parse failures. Skipped for reasoning
            # models (some don't accept response_format alongside reasoning).
            if not is_reasoning:
                kwargs["response_format"] = {"type": "json_object"}
            resp = await _oai().chat.completions.create(**kwargs)
            raw = (resp.choices[0].message.content or "").strip()
        data = _parse_json(raw)
    except Exception as e:
        logger.warning("kol_analysis extract failed for %s: %s", handle, e)
        return {"summary": None, "tickers": [], "model": model,
                "version": ANALYSIS_VERSION, "error": str(e)}

    # Normalize
    tickers = data.get("tickers") or []
    cleaned = []
    valid_actions = {"buy", "sell", "reduce", "bullish", "bearish", "mention"}
    valid_timeframes = {"immediate", "days", "weeks", "months", "unspecified"}
    valid_post_types = {"trade_update", "macro_thesis", "research", "news_recap", "opinion", "other"}
    # Synthetic category tickers — allowed even though they're not real on-chain tokens.
    # Used when the KOL expresses a view on a category without naming specific coins.
    SYNTHETIC_TICKERS = {"MEME", "ALTCOIN", "DEFI", "AI"}

    for t in tickers:
        if not isinstance(t, dict):
            continue
        sym = (t.get("ticker") or "").strip().upper()
        if not sym:
            continue
        # Allow synthetic category tickers; reject anything else > 12 chars
        # (12 chars covers most real tickers; synthetic ones are all ≤ 7 chars)
        if sym not in SYNTHETIC_TICKERS and len(sym) > 12:
            continue
        action = (t.get("action") or "mention").lower()
        if action not in valid_actions:
            action = "mention"
        try:
            conv = float(t.get("conviction") or 0)
        except (TypeError, ValueError):
            conv = 0.0
        conv = max(0.0, min(1.0, conv))
        timeframe = (t.get("timeframe") or "unspecified").lower()
        if timeframe not in valid_timeframes:
            timeframe = "unspecified"
        stance_change = bool(t.get("stance_change", False))
        cleaned.append({
            "ticker": sym,
            "action": action,
            "conviction": round(conv, 2),
            "timeframe": timeframe,
            "stance_change": stance_change,
            "quote": (t.get("quote") or "")[:200],
        })

    # talks_vs_trades_score: new float field (v2). Backward-compat: if the
    # model returns the old boolean talks_vs_trades_flag, convert it so callers
    # that stored the old field still work. Clamp to [0, 1].
    raw_score = data.get("talks_vs_trades_score")
    if raw_score is None:
        # Fallback: old boolean field from v1 responses
        raw_score = 1.0 if bool(data.get("talks_vs_trades_flag", False)) else 0.0
    try:
        tvt_score = round(max(0.0, min(1.0, float(raw_score))), 2)
    except (TypeError, ValueError):
        tvt_score = 0.0

    post_type = (data.get("post_type") or "other").lower()
    if post_type not in valid_post_types:
        post_type = "other"

    tier = _derive_tier(cleaned, tvt_score)

    return {
        "summary": (data.get("summary") or "").strip() or None,
        "post_type": post_type,
        "tickers": cleaned,
        "talks_vs_trades_score": tvt_score,
        # Keep old boolean for any callers that still check it
        "talks_vs_trades_flag": tvt_score >= 0.5,
        # tier mirrors x_analysis' vocabulary (trade_signal / directional /
        # noise) so blog/substack/podcast posts get the same SIGNAL/VIEW UI
        # badges + the "Signals only" filter that Twitter posts already have.
        "tier": tier,
        "model": model,
        "version": ANALYSIS_VERSION,
    }


def _derive_tier(tickers: list[dict], tvt_score: float) -> str:
    """Map kol_analysis output → the trade_signal/directional/noise tiers that
    x_analysis emits directly. Non-Twitter analyzers don't ask the model for a
    tier, so we derive one from the per-ticker conviction + the talks-vs-trades
    (divergence) score. Without this, blog/substack/podcast rows have tier=NULL
    and the "Signals only" filter + SIGNAL/VIEW badges never apply to them.

      * directional ticker = an explicit non-"mention" action (buy/sell/
        reduce/bullish/bearish).
      * trade_signal = high conviction (>= 0.6) on a directional ticker, or a
        strong talks-vs-trades divergence (>= 0.6) — the platform's top signal.
      * directional  = a directional view exists but below the trade_signal bar.
      * noise        = no directional ticker and no notable divergence.
    """
    directional = [t for t in tickers if (t.get("action") or "mention") != "mention"]
    max_conv = max((float(t.get("conviction") or 0) for t in directional), default=0.0)
    if max_conv >= 0.6 or tvt_score >= 0.6:
        return "trade_signal"
    if directional or tvt_score >= 0.5:
        return "directional"
    return "noise"