trumpsignal-backend/app/services/macro/fetchers.py

"""Individual fetchers for each macro indicator.

Each function is an async coroutine that returns a dict shaped like:
    { "value": float | int | None,
      "label": Optional[str],          # only some indicators
      "raw":   <upstream payload> }    # for debugging / re-scoring

Every fetcher MUST tolerate upstream failure — return {"value": None} rather
than raise — so one dead API can't take down the whole snapshot.

Public, free, no-key sources only:

  AHR999                  : derived from BTC daily closes (Binance fapi)
  Altcoin Season Index    : CoinGecko top-50 90-day relative performance
  Fear & Greed            : api.alternative.me/fng (no auth)
  BTC Dominance           : CoinGecko /global
  ETH/BTC Ratio           : Binance kline ETHBTC daily
  Stablecoin Supply       : DeFiLlama /stablecoins
  ETF Net Flow (1d)       : Farside Investors HTML scrape
  BTC Open Interest       : Binance fapi /futures/data/openInterestHist
"""
from __future__ import annotations

import logging
import math
import re
from datetime import datetime, timedelta, timezone
from typing import Any, Optional

import httpx
from app.services.bottom_indicators import ahr999 as compute_ahr999

logger = logging.getLogger(__name__)

# A vanilla User-Agent. CoinGecko + alternative.me + DeFiLlama all happily
# serve "Mozilla/5.0"; some get suspicious of anything that looks bot-like
# (e.g. python-httpx default UA returns 400 on /global).
UA = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15"}
DEFAULT_TIMEOUT = 20


def _none_on_fail(name: str):
    """Decorator: log+swallow exceptions from a fetcher and return {value: None}."""
    def deco(fn):
        async def wrapper(*a, **kw):
            try:
                return await fn(*a, **kw)
            except Exception as exc:
                logger.warning("macro fetch %s failed: %s (%s)",
                               name, type(exc).__name__, exc)
                return {"value": None, "raw": {"error": f"{type(exc).__name__}: {exc}"}}
        return wrapper
    return deco


def _utc_midnight_ms(now: Optional[datetime] = None) -> int:
    dt = now or datetime.now(timezone.utc)
    midnight = dt.replace(hour=0, minute=0, second=0, microsecond=0)
    return int(midnight.timestamp() * 1000)


def _drop_in_progress_daily_klines(rows: list[list], now: Optional[datetime] = None) -> list[list]:
    """Binance daily klines are keyed by OPEN time. If the latest row opened at
    today's 00:00 UTC, that candle is still in progress and should not be used
    for daily snapshots."""
    if not rows:
        return rows
    cutoff = _utc_midnight_ms(now)
    return [row for row in rows if int(row[0]) < cutoff]


def _latest_closed_daily_point(rows: list[dict], now: Optional[datetime] = None) -> Optional[dict]:
    """Same idea as `_drop_in_progress_daily_klines`, but for daily point
    series keyed by `timestamp`."""
    if not rows:
        return None
    cutoff = _utc_midnight_ms(now)
    closed = [row for row in rows if int(row.get("timestamp", 0)) < cutoff]
    return closed[-1] if closed else None


def _parse_farside_latest_total(html: str) -> dict:
    """Extract the most recent dated row from Farside's historical table.

    The all-data table is chronological from oldest to newest, so the first
    date row is NOT the latest one.
    """
    m = re.search(r"<tbody[^>]*>(.*?)</tbody>", html, re.DOTALL | re.IGNORECASE)
    if not m:
        return {"value": None, "raw": {"error": "tbody not found"}}
    body = m.group(1)
    rows = re.findall(r"<tr[^>]*>(.*?)</tr>", body, re.DOTALL | re.IGNORECASE)
    latest: Optional[dict] = None
    for row in rows:
        cells = re.findall(r"<td[^>]*>(.*?)</td>", row, re.DOTALL | re.IGNORECASE)
        if not cells:
            continue
        date_text = re.sub(r"<[^>]+>", "", cells[0]).strip()
        if not re.match(r"\d{1,2}\s+[A-Za-z]+\s+\d{4}", date_text):
            continue
        last_text = re.sub(r"<[^>]+>", "", cells[-1]).strip()
        num = last_text.replace(",", "").replace("(", "-").replace(")", "")
        try:
            millions = float(num)
            row_date = datetime.strptime(date_text, "%d %b %Y").date()
        except ValueError:
            continue
        candidate = {
            "value": round(millions * 1_000_000, 2),
            "raw": {"date": date_text, "millions_usd": millions},
            "_date": row_date,
        }
        if latest is None or candidate["_date"] > latest["_date"]:
            latest = candidate

    if latest is None:
        return {"value": None, "raw": {"error": "no parseable rows"}}
    latest.pop("_date", None)
    return latest


# ── 1. AHR999 ───────────────────────────────────────────────────────────────
# IMPORTANT: this macro-panel AHR999 MUST stay formula-identical to the
# live BTC bottom scanner, otherwise the displayed value can disagree with the
# actual trigger logic. Reuse the canonical implementation from
# app.services.bottom_indicators instead of re-implementing it here.


@_none_on_fail("ahr999")
async def fetch_ahr999() -> dict:
    """Compute the SAME AHR999 used by the BTC bottom scanner.

    Raw input source stays the same (Binance daily BTCUSDT closes); only the
    formula source-of-truth is centralized so the UI cannot drift from the
    trading logic.
    """
    end_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
    start_ms = end_ms - 260 * 24 * 3600 * 1000   # extra buffer after dropping in-progress day
    async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT) as c:
        r = await c.get(
            "https://fapi.binance.com/fapi/v1/klines",
            params={"symbol": "BTCUSDT", "interval": "1d",
                    "startTime": start_ms, "endTime": end_ms, "limit": 300},
        )
        r.raise_for_status()
        rows = _drop_in_progress_daily_klines(r.json())
    closes = [float(row[4]) for row in rows]
    if len(closes) < 200:
        return {"value": None, "raw": {"error": "insufficient candles", "have": len(closes)}}
    price = closes[-1]
    ma200 = sum(closes[-200:]) / 200  # kept in raw for operator intuition only
    days = (datetime.now(timezone.utc) - datetime(2009, 1, 3, tzinfo=timezone.utc)).total_seconds() / 86400
    age_fit = 10 ** (5.84 * math.log10(days) - 17.01)
    ahr = compute_ahr999(closes)
    if ahr is None:
        return {"value": None, "raw": {"error": "ahr999 computation returned null"}}
    return {
        "value": round(ahr, 4),
        "raw": {"price": price, "ma200": round(ma200, 2),
                "age_fit": round(age_fit, 2), "days": round(days, 1)},
    }


# ── 2. Altcoin Season Index (blockchaincenter.net — official source) ─────────
# Scrape the value directly from blockchaincenter.net, which is the canonical
# publisher of this index (90-day window: how many of the top 50 alts beat BTC
# over 90 days). 75+ = altseason, <25 = bitcoin season.
#
# Previous implementation computed the index from CoinGecko /coins/markets
# using a 30d window (CoinGecko doesn't return 90d per-coin data on that
# endpoint). The 30d vs 90d discrepancy caused readings up to ~30 points
# higher than the official index during BTC-dominated markets. Scraping the
# actual page is more reliable than re-implementing the formula.
#
# Fallback: if the page scrape fails, return None (the @_none_on_fail
# decorator handles that gracefully).

_BCC_URL = "https://www.blockchaincenter.net/altcoin-season-index/"
# Regex for the server-rendered value in the Next.js HTML:
# "Season<!-- -->41" or "Season (<!-- -->41" inside the page markup.
_BCC_RE = re.compile(r"Season[^<]{0,20}<!--\s*-->\s*(\d{1,3})")


@_none_on_fail("altcoin_season_index")
async def fetch_altcoin_season_index() -> dict:
    """Fetch the Altcoin Season Index from blockchaincenter.net.

    The site is Next.js SSR — the value is embedded in the initial HTML as a
    server-rendered text node. We parse it with a tight regex and fall back to
    None on any parse failure so the rest of the snapshot is unaffected.
    """
    async with httpx.AsyncClient(
        timeout=DEFAULT_TIMEOUT,
        headers={**UA, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
        follow_redirects=True,
    ) as c:
        r = await c.get(_BCC_URL)
        r.raise_for_status()
        html = r.text

    m = _BCC_RE.search(html)
    if not m:
        return {"value": None, "raw": {"error": "regex did not match", "url": _BCC_URL}}

    value = int(m.group(1))
    if not 0 <= value <= 100:
        return {"value": None, "raw": {"error": f"parsed value out of range: {value}"}}

    return {
        "value": float(value),
        "raw": {"source": "blockchaincenter.net", "window": "90d", "parsed": value},
    }


# ── 3. Fear & Greed (alternative.me) ────────────────────────────────────────


@_none_on_fail("fear_greed")
async def fetch_fear_greed() -> dict:
    async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT, headers=UA) as c:
        r = await c.get("https://api.alternative.me/fng/?limit=1")
        r.raise_for_status()
        data = r.json()
    item = (data.get("data") or [None])[0]
    if not item:
        return {"value": None, "raw": data}
    return {
        "value": int(item.get("value", 0)),
        "label": item.get("value_classification"),
        "raw": item,
    }


# ── 4. BTC Dominance (CoinGecko /global) ────────────────────────────────────


@_none_on_fail("btc_dominance")
async def fetch_btc_dominance() -> dict:
    async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT, headers=UA) as c:
        r = await c.get("https://api.coingecko.com/api/v3/global")
        r.raise_for_status()
        data = r.json()
    pct = (data.get("data", {}).get("market_cap_percentage", {}) or {}).get("btc")
    if pct is None:
        return {"value": None, "raw": data}
    return {"value": round(float(pct), 2),
            "raw": {"total_mcap_usd": data["data"]["total_market_cap"].get("usd")}}


# ── 5. ETH/BTC Ratio (Binance ETHBTC daily) ──────────────────────────────────


@_none_on_fail("eth_btc_ratio")
async def fetch_eth_btc_ratio() -> dict:
    async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT) as c:
        r = await c.get(
            "https://fapi.binance.com/fapi/v1/klines",
            params={"symbol": "ETHBTC", "interval": "1d", "limit": 3},
        )
        # fapi may 404 ETHBTC; fall back to spot kline endpoint via data-api host.
        if r.status_code == 400 or r.status_code == 404:
            r = await c.get(
                "https://data-api.binance.vision/api/v3/klines",
                params={"symbol": "ETHBTC", "interval": "1d", "limit": 3},
            )
        r.raise_for_status()
        rows = _drop_in_progress_daily_klines(r.json())
    if not rows:
        return {"value": None, "raw": rows}
    close = float(rows[-1][4])
    return {"value": round(close, 6), "raw": {"close": close, "n_rows": len(rows)}}


# ── 6. Stablecoin Total Supply (DeFiLlama) ───────────────────────────────────


@_none_on_fail("stablecoin_supply")
async def fetch_stablecoin_supply() -> dict:
    async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT, headers=UA) as c:
        r = await c.get(
            "https://stablecoins.llama.fi/stablecoins",
            params={"includePrices": "true"},
        )
        r.raise_for_status()
        data = r.json()
    # Sum circulating peggedUSD across all stables.
    total = 0.0
    for stable in data.get("peggedAssets", []):
        circ = stable.get("circulating", {}).get("peggedUSD")
        if isinstance(circ, (int, float)):
            total += float(circ)
    if total <= 0:
        return {"value": None, "raw": {"error": "no peggedUSD totals found"}}
    return {"value": round(total, 2),
            "raw": {"n_stables": len(data.get("peggedAssets", []))}}


# ── 7. BTC Spot ETF Net Flow 1d (Farside) ────────────────────────────────────
# Farside doesn't have a JSON API but their daily flow page is parseable. We
# pull the most recent row from the All ETFs sum.


@_none_on_fail("etf_flow")
async def fetch_etf_flow() -> dict:
    async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT, headers=UA,
                                 follow_redirects=True) as c:
        r = await c.get("https://farside.co.uk/bitcoin-etf-flow-all-data/")
        r.raise_for_status()
    return _parse_farside_latest_total(r.text)


# ── 8. BTC Open Interest (Binance fapi) ──────────────────────────────────────


@_none_on_fail("btc_open_interest")
async def fetch_btc_open_interest() -> dict:
    async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT) as c:
        r = await c.get(
            "https://fapi.binance.com/futures/data/openInterestHist",
            params={"symbol": "BTCUSDT", "period": "1d", "limit": 4},
        )
        r.raise_for_status()
        rows = r.json()
    latest = _latest_closed_daily_point(rows)
    if not latest:
        return {"value": None, "raw": rows}
    notional = float(latest.get("sumOpenInterestValue", 0))
    return {"value": round(notional, 2),
            "raw": {"contracts": float(latest.get("sumOpenInterest", 0)),
                    "timestamp": latest.get("timestamp")}}