""" Batch re-analyzer — runs Claude analysis on posts that were backfilled without AI scoring (ai_confidence == 0). Designed to run as a one-shot background task. Processes posts oldest-first, 1 per second, so 479 posts ≈ 8 minutes. Progress is logged at every batch. Usage (via dev endpoint POST /api/dev/reanalyze): curl -X POST "http://localhost:8000/api/dev/reanalyze?limit=500&dry_run=false" """ import asyncio import logging from datetime import datetime, timezone from typing import Optional from sqlalchemy import select logger = logging.getLogger(__name__) # Global so the HTTP endpoint can check progress without storing state externally. _reanalyze_state: dict = { "running": False, "processed": 0, "updated": 0, "errors": 0, "total": 0, "started_at": None, "finished_at": None, } def get_state() -> dict: return dict(_reanalyze_state) async def reanalyze_unscored( db_session_factory, limit: int = 500, dry_run: bool = False, delay_secs: float = 1.0, legacy_signals: bool = False, model: Optional[str] = None, ) -> dict: """ Find all posts with ai_confidence == 0, run analyze_post on each, and write results back to the DB. Args: limit: Maximum number of posts to process in this run. dry_run: If True, analyze but do NOT write to DB (for testing). delay_secs: Pause between API calls to avoid rate-limiting. Returns: Summary dict with processed/updated/errors counts. """ global _reanalyze_state if _reanalyze_state["running"]: logger.warning("reanalyze already in progress, skipping") return _reanalyze_state _reanalyze_state = { "running": True, "processed": 0, "updated": 0, "errors": 0, "total": 0, "started_at": datetime.now(timezone.utc).isoformat(), "finished_at": None, } from app.models import Post from app.services.analysis import analyze_post try: # ── Fetch posts needing (re-)analysis ──────────────────────────── # Two cases: # 1. Never analyzed (ai_confidence == 0) # 2. Analyzed before target_asset column was added (has buy/short # signal but target_asset IS NULL). Pass legacy_signals=True # to target ONLY these, bypassing unscored posts. from sqlalchemy import or_, and_ if legacy_signals: condition = and_( Post.signal.in_(["buy", "short"]), Post.target_asset == None, # noqa: E711 ) else: # Use analysis_version IS NULL (not ai_confidence==0) so that # posts already analyzed as "hold" (conf=0, version set) are not # re-run every time. Only truly unanalyzed posts are targeted. condition = or_( Post.analysis_version == None, # noqa: E711 and_( Post.signal.in_(["buy", "short"]), Post.target_asset == None, # noqa: E711 ) ) async with db_session_factory() as db: rows = await db.execute( select(Post.id, Post.text) .where(condition) .order_by(Post.published_at.asc()) .limit(limit) ) posts_to_do = rows.all() # list of (id, text) tuples _reanalyze_state["total"] = len(posts_to_do) logger.info("Reanalyze: %d posts queued (limit=%d, dry_run=%s)", len(posts_to_do), limit, dry_run) if not posts_to_do: _reanalyze_state["running"] = False _reanalyze_state["finished_at"] = datetime.now(timezone.utc).isoformat() return _reanalyze_state # ── Process each post ───────────────────────────────────────────── for post_id, text in posts_to_do: _reanalyze_state["processed"] += 1 try: analysis = await analyze_post(text, model=model) # caller decides quality vs speed if not dry_run: async with db_session_factory() as db: result = await db.execute( select(Post).where(Post.id == post_id) ) post = result.scalar_one_or_none() if post is None: continue post.sentiment = analysis["sentiment"] post.signal = analysis.get("signal") post.ai_confidence = analysis["confidence"] post.ai_reasoning = analysis.get("reasoning") post.prefilter_reason = analysis.get("prefilter_reason") post.analysis_version = analysis.get("analysis_version") post.relevant = analysis["relevant"] post.price_impact_asset = analysis["asset"] if analysis["relevant"] else None post.target_asset = analysis.get("target_asset") post.category = analysis.get("category") post.expected_move_pct = analysis.get("expected_move_pct") await db.commit() _reanalyze_state["updated"] += 1 if _reanalyze_state["processed"] % 20 == 0: logger.info( "Reanalyze progress: %d/%d processed, %d updated, %d errors", _reanalyze_state["processed"], _reanalyze_state["total"], _reanalyze_state["updated"], _reanalyze_state["errors"], ) except Exception as exc: _reanalyze_state["errors"] += 1 logger.error("Reanalyze error on post %d: %s", post_id, exc) # Rate-limit: wait between API calls if delay_secs > 0: await asyncio.sleep(delay_secs) except Exception as exc: logger.error("Reanalyze fatal error: %s", exc) finally: _reanalyze_state["running"] = False _reanalyze_state["finished_at"] = datetime.now(timezone.utc).isoformat() logger.info( "Reanalyze finished: %d processed, %d updated, %d errors", _reanalyze_state["processed"], _reanalyze_state["updated"], _reanalyze_state["errors"], ) return _reanalyze_state