""" Trump Truth Social scraper — CNN public archive Source: https://ix.cnn.io/data/truth-social/truth_archive.json Updated every ~5 minutes by CNN. """ import hashlib import html import logging import re from datetime import datetime, timezone from typing import Optional import httpx from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.models import Post from app.services.analysis import analyze_post from app.services.price_store import price_store from app.ws.manager import manager logger = logging.getLogger(__name__) ARCHIVE_URL = "https://ix.cnn.io/data/truth-social/truth_archive.json" def _strip_html(text: str) -> str: text = re.sub(r"<[^>]+>", " ", text) text = html.unescape(text) return re.sub(r"\s+", " ", text).strip() def _parse_dt(iso: str) -> datetime: try: return datetime.fromisoformat(iso.replace("Z", "+00:00")).replace(tzinfo=None) except Exception: return datetime.now(timezone.utc).replace(tzinfo=None) async def _fetch_archive() -> Optional[list]: headers = { "User-Agent": "Mozilla/5.0 (compatible; TrumpSignal/1.0)", "Accept": "application/json", } try: async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: resp = await client.get(ARCHIVE_URL, headers=headers) resp.raise_for_status() return resp.json() except Exception as exc: logger.error("Failed to fetch CNN archive: %s", exc) return None async def _process_entry(entry: dict, db: AsyncSession) -> Optional[Post]: external_id = hashlib.md5(str(entry["id"]).encode()).hexdigest() result = await db.execute(select(Post).where(Post.external_id == external_id)) if result.scalar_one_or_none(): return None text = _strip_html(entry.get("content") or "").strip() if not text: return None published_at = _parse_dt(entry.get("created_at", "")) analysis = await analyze_post(text) asset = analysis["asset"] price_impact_m5 = price_impact_m15 = price_impact_m1h = price_at_post = None if asset and analysis["relevant"]: price_at_post = price_store.get_price_at(asset, published_at) price_impact_m5 = price_store.get_pct_change(asset, published_at, 5) price_impact_m15 = price_store.get_pct_change(asset, published_at, 15) price_impact_m1h = price_store.get_pct_change(asset, published_at, 60) post = Post( external_id=external_id, text=text, source="truth", published_at=published_at, sentiment=analysis["sentiment"], signal=analysis.get("signal"), ai_confidence=analysis["confidence"], ai_reasoning=analysis.get("reasoning"), relevant=analysis["relevant"], price_impact_asset=asset if analysis["relevant"] else None, price_impact_m5=price_impact_m5, price_impact_m15=price_impact_m15, price_impact_m1h=price_impact_m1h, price_at_post=price_at_post, ) db.add(post) await db.flush() return post def _post_to_ws_payload(post: Post) -> dict: price_impact = None if post.price_impact_asset and post.price_at_post is not None: price_impact = { "asset": post.price_impact_asset, "m5": post.price_impact_m5 or 0.0, "m15": post.price_impact_m15 or 0.0, "m1h": post.price_impact_m1h or 0.0, "price_at_post": post.price_at_post, } return { "type": "new_post", "post": { "id": post.id, "text": post.text, "source": post.source, "published_at": post.published_at.isoformat(), "sentiment": post.sentiment, "ai_confidence": post.ai_confidence, "relevant": post.relevant, "price_impact": price_impact, }, } async def poll_truth_social(db_session_factory) -> None: logger.info("Polling CNN Truth Social archive...") entries = await _fetch_archive() if not entries: return # Only process the latest 50 entries each poll (archive has 30k+ posts) recent = entries[:50] logger.info("Checking %d recent entries...", len(recent)) async with db_session_factory() as db: try: new_posts = [] for entry in recent: try: post = await _process_entry(entry, db) if post: new_posts.append(post) except Exception as exc: logger.error("Error processing entry %s: %s", entry.get("id"), exc) if new_posts: await db.commit() for post in new_posts: await manager.broadcast(_post_to_ws_payload(post)) logger.info("Saved new post id=%d: %s", post.id, post.text[:60]) try: from app.services.bot_engine import process_post await process_post(post, db) except Exception as exc: logger.error("process_post failed for post %d: %s", post.id, exc) else: logger.info("No new posts found.") except Exception as exc: logger.error("Transaction error: %s", exc) await db.rollback() async def backfill_history(db_session_factory, limit: int = 500) -> None: """One-time backfill of historical posts (no Claude analysis, no price impact).""" logger.info("Starting historical backfill (limit=%d)...", limit) entries = await _fetch_archive() if not entries: logger.error("Backfill failed: could not fetch archive") return to_process = entries[:limit] saved = 0 async with db_session_factory() as db: try: for entry in to_process: external_id = hashlib.md5(str(entry["id"]).encode()).hexdigest() result = await db.execute(select(Post).where(Post.external_id == external_id)) if result.scalar_one_or_none(): continue text = _strip_html(entry.get("content") or "").strip() if not text: continue post = Post( external_id=external_id, text=text, source="truth", published_at=_parse_dt(entry.get("created_at", "")), sentiment="neutral", ai_confidence=0, relevant=False, ) db.add(post) saved += 1 await db.commit() logger.info("Backfill complete: saved %d posts", saved) except Exception as exc: logger.error("Backfill error: %s", exc) await db.rollback()