75 lines
3.3 KiB
Python
75 lines
3.3 KiB
Python
"""Current-chat review replay is offline, complete and text-free at its public boundary."""
|
|
import copy
|
|
import importlib.util
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
PATH = Path(__file__).resolve().parents[1] / "experiments/P0-pilot/assistant_diagnostic.py"
|
|
SPEC = importlib.util.spec_from_file_location("assistant_diagnostic", PATH)
|
|
DIAG = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(DIAG)
|
|
|
|
ITEM = {"id": "synthetic", "source_text": "Synthetic private prose.",
|
|
"lean_code": "theorem t (n : Nat) : n = n := rfl", "verified": True}
|
|
ASSESSMENT = {"records": [{"id": "synthetic", "blind_id": "P01", "source": {
|
|
"roles": [{"kind": "object", "symbols": ["n"], "description": "Private object wording",
|
|
"keywords": ["natural"]},
|
|
{"kind": "goal", "description": "Private goal wording", "keywords": ["equal"]}],
|
|
"goal_has_exists": False},
|
|
"judge": {"verdict": "faithful", "faithful": True, "errors": [],
|
|
"reason": "Private reviewer explanation"}}]}
|
|
|
|
|
|
def test_offline_replay_and_text_free_public_output(monkeypatch):
|
|
def no_network(*args, **kwargs):
|
|
raise AssertionError("Replay must not call an API")
|
|
|
|
monkeypatch.setattr("urllib.request.urlopen", no_network)
|
|
rows = DIAG.evaluate([ITEM], {"P01": "synthetic"}, ASSESSMENT)
|
|
assert len(rows) == 1 and rows[0]["errors"] == {}
|
|
public = DIAG.public_items(rows, {"P01": {"verdict": "faithful"}})
|
|
assert public[0]["assistant_verdict"] == "faithful"
|
|
assert not any(word in str(public) for word in ["Private", "Synthetic", "explanation"])
|
|
assert all(key not in public[0] for key in ["source", "judge", "reason", "lean_code", "source_text"])
|
|
|
|
|
|
def test_duplicate_or_missing_assessments_are_rejected():
|
|
assessment = copy.deepcopy(ASSESSMENT)
|
|
assessment["records"] *= 2
|
|
with pytest.raises(ValueError, match="Duplicate"):
|
|
DIAG.evaluate([ITEM], {"P01": "synthetic"}, assessment)
|
|
with pytest.raises(ValueError, match="same cohort"):
|
|
DIAG.evaluate([ITEM], {"P01": "synthetic"}, {"records": []})
|
|
|
|
|
|
def test_mismatched_item_and_missing_source_are_rejected():
|
|
assessment = copy.deepcopy(ASSESSMENT)
|
|
assessment["records"][0]["id"] = "different"
|
|
with pytest.raises(ValueError, match="mismatched"):
|
|
DIAG.evaluate([ITEM], {"P01": "synthetic"}, assessment)
|
|
with pytest.raises(ValueError, match="Missing source"):
|
|
DIAG.evaluate([{**ITEM, "source_text": ""}], {"P01": "synthetic"}, ASSESSMENT)
|
|
|
|
|
|
def test_judge_cannot_disagree_with_its_boolean_flag():
|
|
assessment = copy.deepcopy(ASSESSMENT)
|
|
assessment["records"][0]["judge"]["faithful"] = False
|
|
with pytest.raises(ValueError, match="Invalid authored judgment"):
|
|
DIAG.evaluate([ITEM], {"P01": "synthetic"}, assessment)
|
|
|
|
|
|
def test_invalid_source_schema_is_rejected():
|
|
assessment = copy.deepcopy(ASSESSMENT)
|
|
assessment["records"][0]["source"]["roles"] = []
|
|
with pytest.raises(ValueError, match="goal"):
|
|
DIAG.evaluate([ITEM], {"P01": "synthetic"}, assessment)
|
|
|
|
|
|
def test_agreement_is_explicitly_not_independent_and_has_no_gate():
|
|
rows = DIAG.evaluate([ITEM], {"P01": "synthetic"}, ASSESSMENT)
|
|
result = DIAG.agreement(rows, {"P01": {"verdict": "faithful"}})
|
|
assert result["metrics_are_non_independent_reference_agreement"] is True
|
|
assert "decision" not in result
|
|
assert DIAG.finite(float("nan")) is None
|