96 lines
6.3 KiB
Python
96 lines
6.3 KiB
Python
"""Cross-check collected public snapshots against same-edition official programs."""
|
|
import argparse
|
|
from collections import Counter
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from paper_atlas.db.store import author_keys,normalized
|
|
from paper_atlas.db.submissions import identities_for_row,timestamp
|
|
|
|
|
|
def load(path,source,year):
|
|
path=Path(path);manifest=json.loads(path.with_suffix('.manifest.json').read_text())
|
|
minimum=2 if source=='iclr_program' else 1
|
|
if manifest.get('parser_version',0)<minimum:raise ValueError('Recollect outdated parser output')
|
|
scope={'source':source,'venue':'ICLR','year':year,'track':'main'}
|
|
if any(manifest.get(key)!=value for key,value in scope.items()):raise ValueError('Manifest scope mismatch')
|
|
rows={};external=set();numbers={};digest=hashlib.sha256()
|
|
with path.open('rb') as stream:
|
|
for line in stream:
|
|
digest.update(line);row=json.loads(line)
|
|
if any(row.get(key)!=value for key,value in scope.items()):raise ValueError('Row scope mismatch')
|
|
forum=(row.get('identifiers') or {}).get('openreview')
|
|
ids=identities_for_row(row)
|
|
if not forum or ('openreview',forum) not in ids:raise ValueError('Real forum identity required for this audit')
|
|
if forum in rows or row['external_id'] in external:raise ValueError('Repeated submission identity')
|
|
for scheme,value in ids:
|
|
if scheme=='iclr_number':
|
|
if value in numbers and numbers[value]!=forum:raise ValueError('One submission number points to different forums')
|
|
numbers[value]=forum
|
|
if source=='iclr_program' and row['status']!='accepted':raise ValueError('Official program row lacks accepted status')
|
|
if source=='openreview_snapshot':
|
|
if not row.get('metadata',{}).get('public_readers_verified'):raise ValueError('Missing public submission evidence')
|
|
events=row.get('decision_events')
|
|
if not isinstance(events,list):raise ValueError('Missing explicit decision evidence array')
|
|
if row['status']!='unknown' and not any(e['status']==row['status'] for e in events):raise ValueError('Missing summary decision evidence')
|
|
for event in events:
|
|
if not event.get('source_decision_id') or not event.get('authority',{}).get('public_readers_verified'):
|
|
raise ValueError('Missing public decision authority')
|
|
rows[forum]=row;external.add(row['external_id'])
|
|
if not rows or len(rows)!=manifest['count']:raise ValueError('Manifest count mismatch')
|
|
missing=sum(not bool((row.get('abstract') or '').strip()) for row in rows.values())
|
|
if missing!=manifest.get('missing_abstract_count'):raise ValueError('Abstract coverage mismatch')
|
|
return rows,manifest,digest.hexdigest()
|
|
|
|
|
|
def audit_year(root,year):
|
|
program_path=root/f'iclr-program-{year}-main.jsonl'
|
|
snapshot_path=root/f'openreview-snapshot-{year}-main.jsonl'
|
|
program,pm,ph=load(program_path,'iclr_program',year)
|
|
snapshot,sm,sh=load(snapshot_path,'openreview_snapshot',year)
|
|
common=program.keys()&snapshot.keys();conflicts=[];text_changes=[]
|
|
for forum in sorted(common):
|
|
p,s=program[forum],snapshot[forum]
|
|
if s['status']!='accepted':conflicts.append({'forum':forum,'official_program':'accepted','snapshot_status':s['status']})
|
|
same_title=normalized(p['title'])==normalized(s['title'])
|
|
same_abstract=(p.get('abstract') or '').strip()==(s.get('abstract') or '').strip()
|
|
overlap=len(author_keys(p.get('authors',[]))&author_keys(s.get('authors',[])))
|
|
if not same_title or not same_abstract:
|
|
text_changes.append({'forum':forum,'same_normalized_title':same_title,'same_abstract':same_abstract,'shared_public_authors':overlap})
|
|
statuses=Counter(row['status'] for row in snapshot.values())
|
|
if dict(statuses)!=sm.get('submission_status_counts'):raise ValueError('Snapshot status counts mismatch')
|
|
report={'year':year,'program_records':len(program),'snapshot_records':len(snapshot),'shared_forums':len(common),
|
|
'snapshot_status_counts':dict(statuses),'snapshot_missing_abstracts':sm['missing_abstract_count'],
|
|
'snapshot_observation_upper_bound':sm.get('snapshot_observation_upper_bound'),
|
|
'program_only_forums':sorted(program.keys()-snapshot.keys()),
|
|
'snapshot_accepted_not_in_program':sorted(forum for forum,row in snapshot.items() if row['status']=='accepted' and forum not in program),
|
|
'program_snapshot_status_differences':conflicts,'shared_forum_text_differences':text_changes,
|
|
'unmatched_public_decisions':sm.get('unmatched_public_decisions',0),
|
|
'interpretation':'Snapshot gaps and unknowns are not rejections. Strong forum identity links events; historical observations and text variants remain preserved.'}
|
|
return report,(str(program_path),ph),(str(snapshot_path),sh)
|
|
|
|
|
|
def audit(root,years,report_path):
|
|
root=Path(root).resolve();reports=[];program={};snapshots={}
|
|
for year in sorted(set(years)):
|
|
report,p,s=audit_year(root,year);reports.append(report);program[p[0]]=p[1];snapshots[s[0]]=s[1]
|
|
result={'checked_at':timestamp(),'years':reports,'backfill_queue':{'files':list(program),'file_sha256':program},
|
|
'snapshot_files':{'files':list(snapshots),'file_sha256':snapshots}}
|
|
target=Path(report_path);part=target.with_suffix('.part')
|
|
part.write_text(json.dumps(result,ensure_ascii=False,indent=2));part.replace(target)
|
|
queue=target.with_name(target.stem+'-backfill-queue.json')
|
|
part=queue.with_suffix('.part');part.write_text(json.dumps(result['backfill_queue'],indent=2));part.replace(queue)
|
|
print(json.dumps({'report':str(target),'backfill_queue':str(queue),'years':[{'year':r['year'],'snapshot_status_counts':r['snapshot_status_counts'],'program_only':len(r['program_only_forums']),'status_differences':len(r['program_snapshot_status_differences'])} for r in reports]}),flush=True)
|
|
return result
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--root',required=True);parser.add_argument('--report',required=True)
|
|
parser.add_argument('--year',type=int,action='append',required=True);args=parser.parse_args(argv)
|
|
audit(args.root,args.year,args.report)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|