fix: de-risk launch_seed, monitor price feeds, enforce single-process
Three pre-launch audit findings: launch_seed.py — destructive cleanup is now OPT-IN Previously `launch_seed.py --yes` truncated KOL history (divergence / holdings / posts older than 30d) by default — an accidental run erased unrecoverable data, and abort_if_live_user_state() only guards on subscriptions/bindings, not KOL history. Now nothing is deleted unless --wipe is passed; the safe path is --seed-only (pure fetch). Bare/--yes without --wipe refuses and prints guidance. /api/health/deep — monitor the price feeds, not just scrapers The deep healthcheck only watched the (redundant) Trump scrapers + DB, so a dead Binance/HL price feed — which silently stops ALL tp_sl_monitor stop-loss / take-profit firing on live trades — left health green. Added per-feed liveness (binance.last_tick_at, hl_price_feed.last_tick_at) with a 180s boot grace so startup doesn't false-503. Body now includes price_feeds[]. Single-process enforcement (multi-worker safety) The backend is single-process by design (in-memory scheduler, replay cache, tp_sl table, price_store). systemd unit lacked the --workers 1 + rationale that supervisor.conf already had; added it. Added a runtime advisory file lock (app.main._acquire_singleton_lock): only the leader starts background tasks; extra workers serve HTTP reads only and log CRITICAL. health/deep now reports is_leader so the misconfig is visible to monitors. 72 tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
+100
-1
@@ -1,5 +1,7 @@
|
||||
import asyncio
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
@@ -41,16 +43,74 @@ logging.basicConfig(
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
_binance_task: Optional[asyncio.Task] = None
|
||||
_hl_price_task: Optional[asyncio.Task] = None
|
||||
_telegram_task: Optional[asyncio.Task] = None
|
||||
_scheduler: Optional[AsyncIOScheduler] = None
|
||||
|
||||
# Process boot time — used by /api/health/deep to grant feeds a startup grace
|
||||
# period before "no tick yet" counts as a failure (avoids a false 503 at boot).
|
||||
_PROCESS_START_AT = datetime.now(timezone.utc)
|
||||
_FEED_BOOT_GRACE_SEC = 180
|
||||
|
||||
# Single-instance guard. The backend is single-process BY DESIGN (APScheduler,
|
||||
# scanner_state, the replay cache, tp_sl_monitor's open-trade table and
|
||||
# price_store are all in-memory). Running >1 worker silently duplicates the
|
||||
# scheduler, splits TP/SL state, and lets two processes both long-poll the
|
||||
# Telegram token. This advisory file lock lets only ONE process ("the leader")
|
||||
# start background tasks; any extra worker serves HTTP reads only and logs loud.
|
||||
_singleton_lock_fd = None
|
||||
_is_leader: bool = False
|
||||
|
||||
|
||||
def _acquire_singleton_lock() -> bool:
|
||||
"""Return True if this process won the advisory lock and may run background
|
||||
tasks. False means another process/worker already holds it (multi-worker
|
||||
misconfig). The OS releases the lock automatically when the holder exits,
|
||||
so a crashed leader frees it for the next start."""
|
||||
global _singleton_lock_fd
|
||||
path = os.environ.get("SINGLETON_LOCK_PATH", "/tmp/trumpsignal-backend.lock")
|
||||
try:
|
||||
fd = open(path, "w")
|
||||
fcntl.flock(fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
try:
|
||||
fd.write(str(os.getpid()))
|
||||
fd.flush()
|
||||
except OSError:
|
||||
pass
|
||||
_singleton_lock_fd = fd # keep the fd alive for the process lifetime
|
||||
return True
|
||||
except BlockingIOError:
|
||||
# Lock held by another process — this is the multi-worker signal.
|
||||
return False
|
||||
except OSError as exc:
|
||||
# Lock file itself is unusable (perms, missing dir). Fail OPEN so a
|
||||
# lock-path problem never takes the whole service down — but log loudly.
|
||||
logger.warning("Singleton lock unavailable (%s) — proceeding as leader.", exc)
|
||||
return True
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
global _binance_task, _hl_price_task, _telegram_task, _scheduler
|
||||
global _binance_task, _hl_price_task, _telegram_task, _scheduler, _is_leader
|
||||
|
||||
# 0. Single-instance guard. If another worker already holds the lock, this
|
||||
# process must NOT start the scheduler / scrapers / price feeds / Telegram
|
||||
# poller — otherwise scans double-fire, TP/SL state splits across processes,
|
||||
# and two pollers fight over the Telegram token. Serve HTTP reads only.
|
||||
_is_leader = _acquire_singleton_lock()
|
||||
if not _is_leader:
|
||||
logger.critical(
|
||||
"SINGLETON LOCK NOT ACQUIRED — another backend process/worker is "
|
||||
"already the leader. This process will serve HTTP reads ONLY and "
|
||||
"start NO background tasks. You are almost certainly running with "
|
||||
">1 worker, which is UNSUPPORTED (see deploy/supervisor.conf). "
|
||||
"Run uvicorn/gunicorn with --workers 1."
|
||||
)
|
||||
yield
|
||||
return
|
||||
|
||||
# 1. Dev convenience only. Production should rely on Alembic so schema
|
||||
# ownership stays explicit and startup never mutates the DB implicitly.
|
||||
@@ -397,13 +457,52 @@ async def health_deep():
|
||||
# 6× the 15s interval — both sources are silent
|
||||
problems.append(f"scrapers: all stale (freshest={freshest_age}s)")
|
||||
|
||||
# 3. Price-feed liveness — INDEPENDENT of the scrapers. A dead price feed
|
||||
# means tp_sl_monitor never fires: stop-loss / take-profit / trailing stops
|
||||
# silently stop protecting live trades. This is a real-money incident the
|
||||
# scraper check above would completely miss, so it gets its own per-feed gate.
|
||||
from app.services import binance as _binance
|
||||
from app.services import hl_price_feed as _hl_feed
|
||||
|
||||
boot_age = int((now - _PROCESS_START_AT).total_seconds())
|
||||
in_grace = boot_age < _FEED_BOOT_GRACE_SEC
|
||||
# (feed name, last_tick_at, max age before stale). Binance ticks multiple
|
||||
# times/sec for BTC/ETH; HL feed polls every 2s. Thresholds are generous.
|
||||
feeds = [
|
||||
("binance_ws", _binance.last_tick_at, 120),
|
||||
("hl_price_feed", _hl_feed.last_tick_at, 60),
|
||||
]
|
||||
feed_status = []
|
||||
for name, last, max_age in feeds:
|
||||
age = int((now - last).total_seconds()) if last else None
|
||||
feed_status.append({
|
||||
"name": name,
|
||||
"last_tick": last.isoformat() if last else None,
|
||||
"age_sec": age,
|
||||
"max_age_sec": max_age,
|
||||
})
|
||||
if last is None:
|
||||
# Never ticked: only a problem once past the boot grace window
|
||||
# (otherwise we'd 503 for the first ~3 min of every restart).
|
||||
if not in_grace:
|
||||
problems.append(f"{name}: no tick since boot ({boot_age}s ago)")
|
||||
elif age is not None and age > max_age:
|
||||
problems.append(f"{name}: stale ({age}s > {max_age}s)")
|
||||
|
||||
# A follower (lost the singleton lock) runs no background tasks — surface it
|
||||
# explicitly so the 503 it already triggers has an obvious root cause.
|
||||
if not _is_leader:
|
||||
problems.append("instance: not the singleton leader (>1 worker?) — no background tasks running")
|
||||
|
||||
body = {
|
||||
"status": "ok" if not problems else "degraded",
|
||||
"now": now.isoformat(),
|
||||
"is_leader": _is_leader,
|
||||
"db_ok": db_ok,
|
||||
"db_error": db_error,
|
||||
"scrapers": source_status,
|
||||
"freshest_age_sec": freshest_age,
|
||||
"price_feeds": feed_status,
|
||||
"problems": problems,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user