first commit
This commit is contained in:
k
@@ -0,0 +1,202 @@
|
||||
"""
|
||||
Trump Truth Social scraper — CNN public archive
|
||||
Source: https://ix.cnn.io/data/truth-social/truth_archive.json
|
||||
Updated every ~5 minutes by CNN.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Post
|
||||
from app.services.analysis import analyze_post
|
||||
from app.services.price_store import price_store
|
||||
from app.ws.manager import manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ARCHIVE_URL = "https://ix.cnn.io/data/truth-social/truth_archive.json"
|
||||
|
||||
|
||||
def _strip_html(text: str) -> str:
|
||||
text = re.sub(r"<[^>]+>", " ", text)
|
||||
text = html.unescape(text)
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def _parse_dt(iso: str) -> datetime:
|
||||
try:
|
||||
return datetime.fromisoformat(iso.replace("Z", "+00:00")).replace(tzinfo=None)
|
||||
except Exception:
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
|
||||
async def _fetch_archive() -> Optional[list]:
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (compatible; TrumpSignal/1.0)",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
|
||||
resp = await client.get(ARCHIVE_URL, headers=headers)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except Exception as exc:
|
||||
logger.error("Failed to fetch CNN archive: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
async def _process_entry(entry: dict, db: AsyncSession) -> Optional[Post]:
|
||||
external_id = hashlib.md5(str(entry["id"]).encode()).hexdigest()
|
||||
|
||||
result = await db.execute(select(Post).where(Post.external_id == external_id))
|
||||
if result.scalar_one_or_none():
|
||||
return None
|
||||
|
||||
text = _strip_html(entry.get("content") or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
published_at = _parse_dt(entry.get("created_at", ""))
|
||||
|
||||
analysis = await analyze_post(text)
|
||||
|
||||
asset = analysis["asset"]
|
||||
price_impact_m5 = price_impact_m15 = price_impact_m1h = price_at_post = None
|
||||
if asset and analysis["relevant"]:
|
||||
price_at_post = price_store.get_price_at(asset, published_at)
|
||||
price_impact_m5 = price_store.get_pct_change(asset, published_at, 5)
|
||||
price_impact_m15 = price_store.get_pct_change(asset, published_at, 15)
|
||||
price_impact_m1h = price_store.get_pct_change(asset, published_at, 60)
|
||||
|
||||
post = Post(
|
||||
external_id=external_id,
|
||||
text=text,
|
||||
source="truth",
|
||||
published_at=published_at,
|
||||
sentiment=analysis["sentiment"],
|
||||
signal=analysis.get("signal"),
|
||||
ai_confidence=analysis["confidence"],
|
||||
ai_reasoning=analysis.get("reasoning"),
|
||||
relevant=analysis["relevant"],
|
||||
price_impact_asset=asset if analysis["relevant"] else None,
|
||||
price_impact_m5=price_impact_m5,
|
||||
price_impact_m15=price_impact_m15,
|
||||
price_impact_m1h=price_impact_m1h,
|
||||
price_at_post=price_at_post,
|
||||
)
|
||||
db.add(post)
|
||||
await db.flush()
|
||||
return post
|
||||
|
||||
|
||||
def _post_to_ws_payload(post: Post) -> dict:
|
||||
price_impact = None
|
||||
if post.price_impact_asset and post.price_at_post is not None:
|
||||
price_impact = {
|
||||
"asset": post.price_impact_asset,
|
||||
"m5": post.price_impact_m5 or 0.0,
|
||||
"m15": post.price_impact_m15 or 0.0,
|
||||
"m1h": post.price_impact_m1h or 0.0,
|
||||
"price_at_post": post.price_at_post,
|
||||
}
|
||||
return {
|
||||
"type": "new_post",
|
||||
"post": {
|
||||
"id": post.id,
|
||||
"text": post.text,
|
||||
"source": post.source,
|
||||
"published_at": post.published_at.isoformat(),
|
||||
"sentiment": post.sentiment,
|
||||
"ai_confidence": post.ai_confidence,
|
||||
"relevant": post.relevant,
|
||||
"price_impact": price_impact,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def poll_truth_social(db_session_factory) -> None:
|
||||
logger.info("Polling CNN Truth Social archive...")
|
||||
entries = await _fetch_archive()
|
||||
if not entries:
|
||||
return
|
||||
|
||||
# Only process the latest 50 entries each poll (archive has 30k+ posts)
|
||||
recent = entries[:50]
|
||||
logger.info("Checking %d recent entries...", len(recent))
|
||||
|
||||
async with db_session_factory() as db:
|
||||
try:
|
||||
new_posts = []
|
||||
for entry in recent:
|
||||
try:
|
||||
post = await _process_entry(entry, db)
|
||||
if post:
|
||||
new_posts.append(post)
|
||||
except Exception as exc:
|
||||
logger.error("Error processing entry %s: %s", entry.get("id"), exc)
|
||||
|
||||
if new_posts:
|
||||
await db.commit()
|
||||
for post in new_posts:
|
||||
await manager.broadcast(_post_to_ws_payload(post))
|
||||
logger.info("Saved new post id=%d: %s", post.id, post.text[:60])
|
||||
try:
|
||||
from app.services.bot_engine import process_post
|
||||
await process_post(post, db)
|
||||
except Exception as exc:
|
||||
logger.error("process_post failed for post %d: %s", post.id, exc)
|
||||
else:
|
||||
logger.info("No new posts found.")
|
||||
except Exception as exc:
|
||||
logger.error("Transaction error: %s", exc)
|
||||
await db.rollback()
|
||||
|
||||
|
||||
async def backfill_history(db_session_factory, limit: int = 500) -> None:
|
||||
"""One-time backfill of historical posts (no Claude analysis, no price impact)."""
|
||||
logger.info("Starting historical backfill (limit=%d)...", limit)
|
||||
entries = await _fetch_archive()
|
||||
if not entries:
|
||||
logger.error("Backfill failed: could not fetch archive")
|
||||
return
|
||||
|
||||
to_process = entries[:limit]
|
||||
saved = 0
|
||||
|
||||
async with db_session_factory() as db:
|
||||
try:
|
||||
for entry in to_process:
|
||||
external_id = hashlib.md5(str(entry["id"]).encode()).hexdigest()
|
||||
result = await db.execute(select(Post).where(Post.external_id == external_id))
|
||||
if result.scalar_one_or_none():
|
||||
continue
|
||||
|
||||
text = _strip_html(entry.get("content") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
post = Post(
|
||||
external_id=external_id,
|
||||
text=text,
|
||||
source="truth",
|
||||
published_at=_parse_dt(entry.get("created_at", "")),
|
||||
sentiment="neutral",
|
||||
ai_confidence=0,
|
||||
relevant=False,
|
||||
)
|
||||
db.add(post)
|
||||
saved += 1
|
||||
|
||||
await db.commit()
|
||||
logger.info("Backfill complete: saved %d posts", saved)
|
||||
except Exception as exc:
|
||||
logger.error("Backfill error: %s", exc)
|
||||
await db.rollback()
|
||||
Reference in New Issue
Block a user