k
207 lines
6.9 KiB
Python
207 lines
6.9 KiB
Python
"""
|
|
Trump Truth Social scraper — CNN public archive
|
|
Source: https://ix.cnn.io/data/truth-social/truth_archive.json
|
|
Updated every ~5 minutes by CNN.
|
|
"""
|
|
|
|
import hashlib
|
|
import html
|
|
import logging
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from typing import Optional
|
|
|
|
import httpx
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models import Post, iso_utc
|
|
from app.services.analysis import analyze_post
|
|
from app.services.price_store import price_store
|
|
from app.ws.manager import manager
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ARCHIVE_URL = "https://ix.cnn.io/data/truth-social/truth_archive.json"
|
|
|
|
|
|
def _strip_html(text: str) -> str:
|
|
text = re.sub(r"<[^>]+>", " ", text)
|
|
text = html.unescape(text)
|
|
return re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
|
def _parse_dt(iso: str) -> datetime:
|
|
try:
|
|
return datetime.fromisoformat(iso.replace("Z", "+00:00")).replace(tzinfo=None)
|
|
except Exception:
|
|
return datetime.now(timezone.utc).replace(tzinfo=None)
|
|
|
|
|
|
async def _fetch_archive() -> Optional[list]:
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (compatible; TrumpSignal/1.0)",
|
|
"Accept": "application/json",
|
|
}
|
|
try:
|
|
async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
|
|
resp = await client.get(ARCHIVE_URL, headers=headers)
|
|
resp.raise_for_status()
|
|
return resp.json()
|
|
except Exception as exc:
|
|
logger.error("Failed to fetch CNN archive: %s", exc)
|
|
return None
|
|
|
|
|
|
async def _process_entry(entry: dict, db: AsyncSession) -> Optional[Post]:
|
|
external_id = hashlib.md5(str(entry["id"]).encode()).hexdigest()
|
|
|
|
result = await db.execute(select(Post).where(Post.external_id == external_id))
|
|
if result.scalar_one_or_none():
|
|
return None
|
|
|
|
text = _strip_html(entry.get("content") or "").strip()
|
|
if not text:
|
|
return None
|
|
|
|
published_at = _parse_dt(entry.get("created_at", ""))
|
|
|
|
analysis = await analyze_post(text)
|
|
|
|
asset = analysis["asset"]
|
|
price_impact_m5 = price_impact_m15 = price_impact_m1h = price_at_post = None
|
|
if asset and analysis["relevant"]:
|
|
price_at_post = price_store.get_price_at(asset, published_at)
|
|
price_impact_m5 = price_store.get_pct_change(asset, published_at, 5)
|
|
price_impact_m15 = price_store.get_pct_change(asset, published_at, 15)
|
|
price_impact_m1h = price_store.get_pct_change(asset, published_at, 60)
|
|
|
|
post = Post(
|
|
external_id=external_id,
|
|
text=text,
|
|
source="truth",
|
|
published_at=published_at,
|
|
sentiment=analysis["sentiment"],
|
|
signal=analysis.get("signal"),
|
|
ai_confidence=analysis["confidence"],
|
|
ai_reasoning=analysis.get("reasoning"),
|
|
prefilter_reason=analysis.get("prefilter_reason"),
|
|
analysis_version=analysis.get("analysis_version"),
|
|
relevant=analysis["relevant"],
|
|
price_impact_asset=asset if analysis["relevant"] else None,
|
|
price_impact_m5=price_impact_m5,
|
|
price_impact_m15=price_impact_m15,
|
|
price_impact_m1h=price_impact_m1h,
|
|
price_at_post=price_at_post,
|
|
)
|
|
db.add(post)
|
|
await db.flush()
|
|
return post
|
|
|
|
|
|
def _post_to_ws_payload(post: Post) -> dict:
|
|
price_impact = None
|
|
if post.price_impact_asset and post.price_at_post is not None:
|
|
price_impact = {
|
|
"asset": post.price_impact_asset,
|
|
"m5": post.price_impact_m5 or 0.0,
|
|
"m15": post.price_impact_m15 or 0.0,
|
|
"m1h": post.price_impact_m1h or 0.0,
|
|
"price_at_post": post.price_at_post,
|
|
}
|
|
return {
|
|
"type": "new_post",
|
|
"post": {
|
|
"id": post.id,
|
|
"text": post.text,
|
|
"source": post.source,
|
|
"published_at": iso_utc(post.published_at),
|
|
"sentiment": post.sentiment,
|
|
"signal": post.signal,
|
|
"ai_confidence": post.ai_confidence,
|
|
"ai_reasoning": post.ai_reasoning,
|
|
"relevant": post.relevant,
|
|
"price_impact": price_impact,
|
|
},
|
|
}
|
|
|
|
|
|
async def poll_truth_social(db_session_factory) -> None:
|
|
logger.info("Polling CNN Truth Social archive...")
|
|
entries = await _fetch_archive()
|
|
if not entries:
|
|
return
|
|
|
|
# Only process the latest 50 entries each poll (archive has 30k+ posts)
|
|
recent = entries[:50]
|
|
logger.info("Checking %d recent entries...", len(recent))
|
|
|
|
async with db_session_factory() as db:
|
|
try:
|
|
new_posts = []
|
|
for entry in recent:
|
|
try:
|
|
post = await _process_entry(entry, db)
|
|
if post:
|
|
new_posts.append(post)
|
|
except Exception as exc:
|
|
logger.error("Error processing entry %s: %s", entry.get("id"), exc)
|
|
|
|
if new_posts:
|
|
await db.commit()
|
|
for post in new_posts:
|
|
await manager.broadcast(_post_to_ws_payload(post))
|
|
logger.info("Saved new post id=%d: %s", post.id, post.text[:60])
|
|
try:
|
|
from app.services.bot_engine import process_post
|
|
await process_post(post, db)
|
|
except Exception as exc:
|
|
logger.error("process_post failed for post %d: %s", post.id, exc)
|
|
else:
|
|
logger.info("No new posts found.")
|
|
except Exception as exc:
|
|
logger.error("Transaction error: %s", exc)
|
|
await db.rollback()
|
|
|
|
|
|
async def backfill_history(db_session_factory, limit: int = 500) -> None:
|
|
"""One-time backfill of historical posts (no Claude analysis, no price impact)."""
|
|
logger.info("Starting historical backfill (limit=%d)...", limit)
|
|
entries = await _fetch_archive()
|
|
if not entries:
|
|
logger.error("Backfill failed: could not fetch archive")
|
|
return
|
|
|
|
to_process = entries[:limit]
|
|
saved = 0
|
|
|
|
async with db_session_factory() as db:
|
|
try:
|
|
for entry in to_process:
|
|
external_id = hashlib.md5(str(entry["id"]).encode()).hexdigest()
|
|
result = await db.execute(select(Post).where(Post.external_id == external_id))
|
|
if result.scalar_one_or_none():
|
|
continue
|
|
|
|
text = _strip_html(entry.get("content") or "").strip()
|
|
if not text:
|
|
continue
|
|
|
|
post = Post(
|
|
external_id=external_id,
|
|
text=text,
|
|
source="truth",
|
|
published_at=_parse_dt(entry.get("created_at", "")),
|
|
sentiment="neutral",
|
|
ai_confidence=0,
|
|
relevant=False,
|
|
)
|
|
db.add(post)
|
|
saved += 1
|
|
|
|
await db.commit()
|
|
logger.info("Backfill complete: saved %d posts", saved)
|
|
except Exception as exc:
|
|
logger.error("Backfill error: %s", exc)
|
|
await db.rollback()
|