Z/trumpsignal-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add daily budget, active window, trade snapshots, and price impact monitor

Browse Source 
- New migrations for daily_budget, active_window, and bottrade snapshot
- Add trumpstruth scraper and price_impact_monitor service
- Expand bot_engine, hyperliquid, recovery, and tp_sl_monitor logic
- Update API/schemas/models for new features

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
k
2026-04-25 16:04:49 +08:00
parent a2c68e2939
commit 4ffcb442fe
20 changed files with 1110 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/scrapers/trumpstruth.py
+155
View File
@@ -0,0 +1,155 @@
"""
Trump Truth Social scraper — trumpstruth.org RSS fallback.
Why a second scraper?
CNN's archive (the primary source) sometimes lags 5–10 minutes behind real
posts. trumpstruth.org publishes an RSS feed of the same Truth Social account
that often updates faster. Running both in parallel and deduping by the
Truth Social `originalId` gives us "min(latency_a, latency_b)" — i.e. whoever
sees the post first wins.
Dedup strategy:
Both CNN and trumpstruth expose the underlying Truth Social post id. We
hash it the same way (md5(str(id))) so the second source is a no-op when
the first already inserted the row.
Source: https://www.trumpstruth.org/feed (RSS 2.0 with custom truth:originalId tag)
"""
import hashlib
import logging
import re
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Optional
import httpx
from app.scrapers.truth_social import _process_entry, _post_to_ws_payload
from app.ws.manager import manager
logger = logging.getLogger(__name__)
FEED_URL = "https://www.trumpstruth.org/feed"
NS = {
"atom": "http://www.w3.org/2005/Atom",
"truth": "https://truthsocial.com/ns",
}
# Liveness — read by /api/health/deep
last_successful_poll_at: Optional[datetime] = None
last_poll_error: Optional[str] = None
async def _fetch_feed() -> Optional[str]:
headers = {
"User-Agent": "Mozilla/5.0 (compatible; TrumpSignal/1.0)",
"Accept": "application/rss+xml, application/xml",
}
try:
async with httpx.AsyncClient(timeout=20, follow_redirects=True) as client:
resp = await client.get(FEED_URL, headers=headers)
resp.raise_for_status()
return resp.text
except Exception as exc:
logger.warning("Failed to fetch trumpstruth.org feed: %s", exc)
return None
_HTML_TAG = re.compile(r"<[^>]+>")
def _to_cnn_shape(item: ET.Element) -> Optional[dict]:
"""Convert one <item> from the RSS feed into the dict shape the existing
`_process_entry` expects (CNN archive format).
Required output keys: id, created_at (ISO), content (HTML)."""
orig_id_el = item.find("truth:originalId", NS)
if orig_id_el is None or not (orig_id_el.text or "").strip():
return None
orig_id = orig_id_el.text.strip()
pub_el = item.find("pubDate")
if pub_el is None or not pub_el.text:
return None
try:
dt = parsedate_to_datetime(pub_el.text)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
else:
dt = dt.astimezone(timezone.utc)
created_iso = dt.isoformat().replace("+00:00", "Z")
except Exception:
return None
desc_el = item.find("description")
content_html = (desc_el.text or "").strip() if desc_el is not None else ""
return {
"id": orig_id,
"created_at": created_iso,
"content": content_html,
}
async def poll_trumpstruth(db_session_factory) -> None:
"""One poll cycle. Called by APScheduler.
Idempotent: posts already inserted by the CNN scraper are skipped via the
`external_id` uniqueness check inside `_process_entry`.
"""
global last_successful_poll_at, last_poll_error
raw = await _fetch_feed()
if raw is None:
last_poll_error = "fetch_feed returned None"
return
try:
root = ET.fromstring(raw)
except ET.ParseError as exc:
logger.warning("trumpstruth RSS parse error: %s", exc)
last_poll_error = f"parse: {exc}"
return
items = root.findall(".//item")
if not items:
last_poll_error = "feed had no <item>"
return
# Same as CNN: process the latest 50 only — keeps poll fast and avoids
# re-scanning the whole feed every 15s.
recent = items[:50]
entries = [e for e in (_to_cnn_shape(it) for it in recent) if e]
async with db_session_factory() as db:
try:
new_posts = []
for entry in entries:
try:
post = await _process_entry(entry, db)
if post:
new_posts.append(post)
except Exception as exc:
logger.error("trumpstruth: error on entry %s: %s",
entry.get("id"), exc)
if new_posts:
await db.commit()
for post in new_posts:
await manager.broadcast(_post_to_ws_payload(post))
logger.info("[trumpstruth] beat CNN — new post id=%d: %s",
post.id, post.text[:60])
try:
from app.services.bot_engine import process_post
await process_post(post, db)
except Exception as exc:
logger.error("process_post failed for post %d: %s",
post.id, exc)
last_successful_poll_at = datetime.now(timezone.utc)
last_poll_error = None
except Exception as exc:
logger.error("trumpstruth transaction error: %s", exc)
last_poll_error = f"transaction: {exc}"
await db.rollback()
app/scrapers/truth_social.py
+45 -11
View File
@@ -24,6 +24,11 @@ logger = logging.getLogger(__name__)
ARCHIVE_URL = "https://ix.cnn.io/data/truth-social/truth_archive.json"
# Liveness tracker — updated every successful poll (even when no new posts).
# Read by /api/health to detect a dead scraper. None = never ran yet.
last_successful_poll_at: Optional[datetime] = None
last_poll_error: Optional[str] = None
def _strip_html(text: str) -> str:
text = re.sub(r"<[^>]+>", " ", text)
@@ -32,8 +37,17 @@ def _strip_html(text: str) -> str:
def _parse_dt(iso: str) -> datetime:
"""Parse Truth Social's ISO timestamp into a naive-UTC datetime.
IMPORTANT: must convert to UTC *before* stripping tzinfo. Otherwise an
input like '2026-04-24T15:07:48-04:00' would be stored as naive
15:07:48 and later mis-read as UTC — a silent 4-hour shift.
"""
try:
return datetime.fromisoformat(iso.replace("Z", "+00:00")).replace(tzinfo=None)
dt = datetime.fromisoformat(iso.replace("Z", "+00:00"))
if dt.tzinfo is not None:
dt = dt.astimezone(timezone.utc)
return dt.replace(tzinfo=None)
except Exception:
return datetime.now(timezone.utc).replace(tzinfo=None)
@@ -69,12 +83,12 @@ async def _process_entry(entry: dict, db: AsyncSession) -> Optional[Post]:
analysis = await analyze_post(text)
asset = analysis["asset"]
price_impact_m5 = price_impact_m15 = price_impact_m1h = price_at_post = None
# Only capture the price AT post time. The m5/m15/m1h peaks are filled in
# asynchronously by price_impact_monitor as the windows elapse — avoids
# recording 0.00% because future candles don't exist yet at entry time.
price_at_post = None
if asset and analysis["relevant"]:
price_at_post = price_store.get_price_at(asset, published_at)
price_impact_m5 = price_store.get_pct_change(asset, published_at, 5)
price_impact_m15 = price_store.get_pct_change(asset, published_at, 15)
price_impact_m1h = price_store.get_pct_change(asset, published_at, 60)
post = Post(
external_id=external_id,
@@ -89,24 +103,38 @@ async def _process_entry(entry: dict, db: AsyncSession) -> Optional[Post]:
analysis_version=analysis.get("analysis_version"),
relevant=analysis["relevant"],
price_impact_asset=asset if analysis["relevant"] else None,
price_impact_m5=price_impact_m5,
price_impact_m15=price_impact_m15,
price_impact_m1h=price_impact_m1h,
price_impact_m5=None, # filled by price_impact_monitor after 5 m
price_impact_m15=None, # filled by price_impact_monitor after 15 m
price_impact_m1h=None, # filled by price_impact_monitor after 1 h
price_at_post=price_at_post,
)
db.add(post)
await db.flush()
# Register with the live peak tracker so it starts watching immediately.
if asset and analysis["relevant"] and price_at_post:
from app.services.price_impact_monitor import register_post
register_post(
post_id=post.id,
asset=asset,
signal=analysis.get("signal"),
entry_price=price_at_post,
published_at=published_at,
)
return post
def _post_to_ws_payload(post: Post) -> dict:
price_impact = None
if post.price_impact_asset and post.price_at_post is not None:
# At broadcast time all windows are open — values are null until
# price_impact_monitor fills them in. Frontend treats null as "pending".
price_impact = {
"asset": post.price_impact_asset,
"m5": post.price_impact_m5 or 0.0,
"m15": post.price_impact_m15 or 0.0,
"m1h": post.price_impact_m1h or 0.0,
"m5": post.price_impact_m5, # None = not yet measured
"m15": post.price_impact_m15,
"m1h": post.price_impact_m1h,
"price_at_post": post.price_at_post,
}
return {
@@ -127,9 +155,11 @@ def _post_to_ws_payload(post: Post) -> dict:
async def poll_truth_social(db_session_factory) -> None:
global last_successful_poll_at, last_poll_error
logger.info("Polling CNN Truth Social archive...")
entries = await _fetch_archive()
if not entries:
last_poll_error = "fetch_archive returned empty"
return
# Only process the latest 50 entries each poll (archive has 30k+ posts)
@@ -159,8 +189,12 @@ async def poll_truth_social(db_session_factory) -> None:
logger.error("process_post failed for post %d: %s", post.id, exc)
else:
logger.info("No new posts found.")
# Mark a successful poll cycle (separate from "found new posts").
last_successful_poll_at = datetime.now(timezone.utc)
last_poll_error = None
except Exception as exc:
logger.error("Transaction error: %s", exc)
last_poll_error = f"transaction_error: {exc}"
await db.rollback()