51 lines
2.3 KiB
Python
51 lines
2.3 KiB
Python
"""JSONL benchmark runner; failed tasks stay in the denominator."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
from lean_agent.cli import options, components
|
|
from lean_agent.core import solve
|
|
|
|
def load_tasks(path):
|
|
tasks = [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
|
|
seen = set()
|
|
for task in tasks:
|
|
if not isinstance(task.get('id'), str) or not isinstance(task.get('statement'), str):
|
|
raise ValueError('Every task needs string id and statement')
|
|
if task['id'] in seen: raise ValueError('Duplicate task id: ' + task['id'])
|
|
seen.add(task['id'])
|
|
if not tasks: raise ValueError('Empty dataset')
|
|
return tasks
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Evaluate Lean agent on trusted JSONL tasks')
|
|
parser.add_argument('dataset', type=Path)
|
|
parser.add_argument('--output', type=Path, required=True, help='New directory for this run')
|
|
options(parser)
|
|
args = parser.parse_args()
|
|
if args.attempts < 1: parser.error('--attempts must be positive')
|
|
tasks = load_tasks(args.dataset)
|
|
provider, verifier = components(args)
|
|
args.output.mkdir(parents=True, exist_ok=False)
|
|
version = subprocess.run([*verifier.command, '--version'], cwd=verifier.cwd,
|
|
capture_output=True, text=True, timeout=30).stdout.strip()
|
|
passed = errors = total_attempts = 0
|
|
with (args.output / 'results.jsonl').open('w') as stream:
|
|
for task in tasks:
|
|
result = solve(provider, verifier, task, args.attempts)
|
|
passed += result['success']
|
|
errors += 'error' in result
|
|
total_attempts += len(result['attempts'])
|
|
stream.write(json.dumps(result, ensure_ascii=False) + '\n')
|
|
stream.flush()
|
|
summary = {'tasks': len(tasks), 'solved': passed, 'solve_rate': passed / len(tasks),
|
|
'errors': errors, 'attempts': total_attempts, 'max_attempts': args.attempts,
|
|
'model': args.model, 'provider': 'command' if args.provider_command else 'http',
|
|
'lean_version': version,
|
|
'dataset_sha256': hashlib.sha256(args.dataset.read_bytes()).hexdigest()}
|
|
(args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')
|
|
print(json.dumps(summary, indent=2))
|
|
|
|
if __name__ == '__main__': main()
|