improve signed reads, crypto hardening, and scraper transport

app/scrapers/trumpstruth.py

@@ -24,10 +24,12 @@ from datetime import datetime, timezone
 from email.utils import parsedate_to_datetime
 from typing import Optional
 
-import httpx
-
-from app.scrapers.truth_social import _process_entry, _post_to_ws_payload
-from app.ws.manager import manager
+from app.scrapers.truth_social import (
+    NOT_MODIFIED,
+    _known_external_ids,
+    _process_entry,
+    dispatch_post,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -41,17 +43,32 @@ NS = {
 last_successful_poll_at: Optional[datetime] = None
 last_poll_error: Optional[str] = None
 
+# Conditional-GET validators (same scheme as truth_social.py — most polls
+# come back 304 and skip the download + XML parse entirely).
+_etag: Optional[str] = None
+_last_modified: Optional[str] = None
 
-async def _fetch_feed() -> Optional[str]:
+
+async def _fetch_feed():
+    """Fetch the RSS body. Returns str, NOT_MODIFIED (304), or None on error."""
+    global _etag, _last_modified
     headers = {
         "User-Agent": "Mozilla/5.0 (compatible; TrumpSignal/1.0)",
         "Accept": "application/rss+xml, application/xml",
     }
+    if _etag:
+        headers["If-None-Match"] = _etag
+    if _last_modified:
+        headers["If-Modified-Since"] = _last_modified
     try:
-        async with httpx.AsyncClient(timeout=20, follow_redirects=True) as client:
-            resp = await client.get(FEED_URL, headers=headers)
-            resp.raise_for_status()
-            return resp.text
+        from app.services.http_client import get_client
+        resp = await get_client().get(FEED_URL, headers=headers, timeout=20)
+        if resp.status_code == 304:
+            return NOT_MODIFIED
+        resp.raise_for_status()
+        _etag = resp.headers.get("etag")
+        _last_modified = resp.headers.get("last-modified")
+        return resp.text
     except Exception as exc:
         # Include type name — httpx often raises bare ConnectError/RemoteProtocolError
         # with empty .args, which formats as just "Failed to fetch ..." with no body.
@@ -105,6 +122,11 @@ async def poll_trumpstruth(db_session_factory) -> None:
     global last_successful_poll_at, last_poll_error
 
     raw = await _fetch_feed()
+    if raw is NOT_MODIFIED:
+        # Feed unchanged — successful cycle, nothing to do.
+        last_successful_poll_at = datetime.now(timezone.utc)
+        last_poll_error = None
+        return
     if raw is None:
         last_poll_error = "fetch_feed returned None"
         return
@@ -128,41 +150,26 @@ async def poll_trumpstruth(db_session_factory) -> None:
 
     async with db_session_factory() as db:
         try:
-            new_posts = []
+            known_ids = await _known_external_ids(entries, db)
+            # Newest-first: commit + dispatch each new post immediately so an
+            # actionable post never queues behind older entries' AI analysis.
+            # dispatch_post is the shared fan-out (WS + Telegram + X + trade)
+            # from truth_social.py — delivery must not depend on which poller
+            # wins the race.
             for entry in entries:
                 try:
-                    post = await _process_entry(entry, db)
+                    post = await _process_entry(entry, db, known_ids)
                     if post:
-                        new_posts.append(post)
+                        await db.commit()
+                        logger.info("[trumpstruth] beat CNN — new post id=%d",
+                                    post.id)
+                        await dispatch_post(post, db)
                 except Exception as exc:
                     logger.error("trumpstruth: error on entry %s: %s",
                                  entry.get("id"), exc)
 
-            if new_posts:
-                await db.commit()
-                for post in new_posts:
-                    await manager.broadcast(_post_to_ws_payload(post))
-                    logger.info("[trumpstruth] beat CNN — new post id=%d: %s",
-                                post.id, post.text[:60])
-                    # Telegram fan-out — matches truth_social.py. Without this,
-                    # whichever poller wins the race determines whether users
-                    # get pushed — flaky 50% delivery.
-                    try:
-                        from app.services.telegram import notify_signal
-                        notify_signal(post)
-                    except Exception as exc:
-                        logger.warning("Telegram notify failed for post %d: %s", post.id, exc)
-                    try:
-                        from app.services.x_poster import notify_x_signal
-                        notify_x_signal(post)
-                    except Exception as exc:
-                        logger.warning("X notify failed for post %d: %s", post.id, exc)
-                    try:
-                        from app.services.bot_engine import process_post
-                        await process_post(post, db)
-                    except Exception as exc:
-                        logger.error("process_post failed for post %d: %s",
-                                     post.id, exc)
+            # Capture any remaining writes (entry-filter stub rows).
+            await db.commit()
             last_successful_poll_at = datetime.now(timezone.utc)
             last_poll_error = None
         except Exception as exc: