trumpsignal-backend/app/services/reanalyze.py

"""
Batch re-analyzer — runs Claude analysis on posts that were backfilled
without AI scoring (ai_confidence == 0).

Designed to run as a one-shot background task. Processes posts oldest-first,
1 per second, so 479 posts ≈ 8 minutes. Progress is logged at every batch.

Usage (via dev endpoint POST /api/dev/reanalyze):
  curl -X POST "http://localhost:8000/api/dev/reanalyze?limit=500&dry_run=false"
"""

import asyncio
import logging
from datetime import datetime, timezone
from typing import Optional

from sqlalchemy import select

logger = logging.getLogger(__name__)

# Global so the HTTP endpoint can check progress without storing state externally.
_reanalyze_state: dict = {
    "running": False,
    "processed": 0,
    "updated": 0,
    "errors": 0,
    "total": 0,
    "started_at": None,
    "finished_at": None,
}


def get_state() -> dict:
    return dict(_reanalyze_state)


async def reanalyze_unscored(
    db_session_factory,
    limit: int = 500,
    dry_run: bool = False,
    delay_secs: float = 1.0,
    legacy_signals: bool = False,
    model: Optional[str] = None,
) -> dict:
    """
    Find all posts with ai_confidence == 0, run analyze_post on each,
    and write results back to the DB.

    Args:
        limit:      Maximum number of posts to process in this run.
        dry_run:    If True, analyze but do NOT write to DB (for testing).
        delay_secs: Pause between API calls to avoid rate-limiting.

    Returns:
        Summary dict with processed/updated/errors counts.
    """
    global _reanalyze_state

    if _reanalyze_state["running"]:
        logger.warning("reanalyze already in progress, skipping")
        return _reanalyze_state

    _reanalyze_state = {
        "running": True,
        "processed": 0,
        "updated": 0,
        "errors": 0,
        "total": 0,
        "started_at": datetime.now(timezone.utc).isoformat(),
        "finished_at": None,
    }

    from app.models import Post
    from app.services.analysis import analyze_post

    try:
        # ── Fetch posts needing (re-)analysis ────────────────────────────
        # Two cases:
        #   1. Never analyzed (ai_confidence == 0)
        #   2. Analyzed before target_asset column was added (has buy/short
        #      signal but target_asset IS NULL). Pass legacy_signals=True
        #      to target ONLY these, bypassing unscored posts.
        from sqlalchemy import or_, and_
        if legacy_signals:
            condition = and_(
                Post.signal.in_(["buy", "short"]),
                Post.target_asset == None,  # noqa: E711
            )
        else:
            # Use analysis_version IS NULL (not ai_confidence==0) so that
            # posts already analyzed as "hold" (conf=0, version set) are not
            # re-run every time. Only truly unanalyzed posts are targeted.
            condition = or_(
                Post.analysis_version == None,  # noqa: E711
                and_(
                    Post.signal.in_(["buy", "short"]),
                    Post.target_asset == None,  # noqa: E711
                )
            )
        async with db_session_factory() as db:
            rows = await db.execute(
                select(Post.id, Post.text)
                .where(condition)
                .order_by(Post.published_at.asc())
                .limit(limit)
            )
            posts_to_do = rows.all()   # list of (id, text) tuples

        _reanalyze_state["total"] = len(posts_to_do)
        logger.info("Reanalyze: %d posts queued (limit=%d, dry_run=%s)",
                    len(posts_to_do), limit, dry_run)

        if not posts_to_do:
            _reanalyze_state["running"] = False
            _reanalyze_state["finished_at"] = datetime.now(timezone.utc).isoformat()
            return _reanalyze_state

        # ── Process each post ─────────────────────────────────────────────
        for post_id, text in posts_to_do:
            _reanalyze_state["processed"] += 1

            try:
                analysis = await analyze_post(text, model=model)  # caller decides quality vs speed

                if not dry_run:
                    async with db_session_factory() as db:
                        result = await db.execute(
                            select(Post).where(Post.id == post_id)
                        )
                        post = result.scalar_one_or_none()
                        if post is None:
                            continue

                        post.sentiment        = analysis["sentiment"]
                        post.signal           = analysis.get("signal")
                        post.ai_confidence    = analysis["confidence"]
                        post.ai_reasoning     = analysis.get("reasoning")
                        post.prefilter_reason = analysis.get("prefilter_reason")
                        post.analysis_version = analysis.get("analysis_version")
                        post.relevant         = analysis["relevant"]
                        post.price_impact_asset = analysis["asset"] if analysis["relevant"] else None
                        post.target_asset     = analysis.get("target_asset")
                        post.category         = analysis.get("category")
                        post.expected_move_pct = analysis.get("expected_move_pct")

                        await db.commit()

                _reanalyze_state["updated"] += 1

                if _reanalyze_state["processed"] % 20 == 0:
                    logger.info(
                        "Reanalyze progress: %d/%d processed, %d updated, %d errors",
                        _reanalyze_state["processed"],
                        _reanalyze_state["total"],
                        _reanalyze_state["updated"],
                        _reanalyze_state["errors"],
                    )

            except Exception as exc:
                _reanalyze_state["errors"] += 1
                logger.error("Reanalyze error on post %d: %s", post_id, exc)

            # Rate-limit: wait between API calls
            if delay_secs > 0:
                await asyncio.sleep(delay_secs)

    except Exception as exc:
        logger.error("Reanalyze fatal error: %s", exc)
    finally:
        _reanalyze_state["running"] = False
        _reanalyze_state["finished_at"] = datetime.now(timezone.utc).isoformat()
        logger.info(
            "Reanalyze finished: %d processed, %d updated, %d errors",
            _reanalyze_state["processed"],
            _reanalyze_state["updated"],
            _reanalyze_state["errors"],
        )

    return _reanalyze_state