Audit public ICLR snapshots against official same-year programs

README.md

@@ -186,6 +186,16 @@ python3 -B collect_openreview_snapshot.py \
 
 该版本是 2026-05-08 发布的第三方历史副本，不代表平台当前状态或全部投稿；缺失决定及公开性筛选均需核验。采集成功也不等于主库已更新，后续应通过单写入协调器导入，并核对同届官方程序与公开决定的差异。
 
+入库前按真实 forum 对照同届官方程序，生成逐年状态、名单缺口、历史决定差异和文本版本差异报告，以及带 SHA-256 的现有程序身份回填队列：
+
+```sh
+python3 -B audit_iclr_snapshot.py --root /data/paper-index/collected \
+  --year 2024 --year 2025 --year 2026 \
+  --report /data/paper-index/collected/iclr-recent-audit.json
+```
+
+该检查要求清单和记录的届次、数量、摘要缺失统计一致，拒绝重复 forum、编号冲突及缺失公开决定证据。报告中的名单差异和 unknown 不会转换成拒稿。不同时间来源的决定保留各自证据，强 forum 标识负责关联同一次投稿；报告通过不等于平台全量覆盖或最终全库去重已经完成。生成的 `*-backfill-queue.json` 仅用于已入库程序记录的身份回填，快照文件仍须由协调器单独导入。
+
 下载异常后先检查对应进程或 CI 任务确已终止，再使用同一参数增加 `--resume`。只有断点身份一致且服务器实际返回正确 Range 响应才续传；不支持时保留断点并报错，不自动重下。正式文件与下载清单均放在仓库外，不提交大型来源包。
 
 ### COLM 与 NeurIPS 官方摘要

audit_iclr_snapshot.py

@@ -0,0 +1,91 @@
+"""Cross-check collected public snapshots against same-edition official programs."""
+import argparse
+from collections import Counter
+import hashlib
+import json
+from pathlib import Path
+
+from index_core import author_keys,normalized
+from submission_evidence import identities_for_row,timestamp
+
+
+def load(path,source,year):
+    path=Path(path);manifest=json.loads(path.with_suffix('.manifest.json').read_text())
+    minimum=2 if source=='iclr_program' else 1
+    if manifest.get('parser_version',0)<minimum:raise ValueError('Recollect outdated parser output')
+    scope={'source':source,'venue':'ICLR','year':year,'track':'main'}
+    if any(manifest.get(key)!=value for key,value in scope.items()):raise ValueError('Manifest scope mismatch')
+    rows={};external=set();numbers={};digest=hashlib.sha256()
+    with path.open('rb') as stream:
+        for line in stream:
+            digest.update(line);row=json.loads(line)
+            if any(row.get(key)!=value for key,value in scope.items()):raise ValueError('Row scope mismatch')
+            forum=(row.get('identifiers') or {}).get('openreview')
+            ids=identities_for_row(row)
+            if not forum or ('openreview',forum) not in ids:raise ValueError('Real forum identity required for this audit')
+            if forum in rows or row['external_id'] in external:raise ValueError('Repeated submission identity')
+            for scheme,value in ids:
+                if scheme=='iclr_number':
+                    if value in numbers and numbers[value]!=forum:raise ValueError('One submission number points to different forums')
+                    numbers[value]=forum
+            if source=='iclr_program' and row['status']!='accepted':raise ValueError('Official program row lacks accepted status')
+            if source=='openreview_snapshot':
+                if not row.get('metadata',{}).get('public_readers_verified'):raise ValueError('Missing public submission evidence')
+                events=row.get('decision_events')
+                if not isinstance(events,list):raise ValueError('Missing explicit decision evidence array')
+                if row['status']!='unknown' and not any(e['status']==row['status'] for e in events):raise ValueError('Missing summary decision evidence')
+                for event in events:
+                    if not event.get('source_decision_id') or not event.get('authority',{}).get('public_readers_verified'):
+                        raise ValueError('Missing public decision authority')
+            rows[forum]=row;external.add(row['external_id'])
+    if not rows or len(rows)!=manifest['count']:raise ValueError('Manifest count mismatch')
+    missing=sum(not bool((row.get('abstract') or '').strip()) for row in rows.values())
+    if missing!=manifest.get('missing_abstract_count'):raise ValueError('Abstract coverage mismatch')
+    return rows,manifest,digest.hexdigest()
+
+
+def audit_year(root,year):
+    program_path=root/f'iclr-program-{year}-main.jsonl'
+    snapshot_path=root/f'openreview-snapshot-{year}-main.jsonl'
+    program,pm,ph=load(program_path,'iclr_program',year)
+    snapshot,sm,sh=load(snapshot_path,'openreview_snapshot',year)
+    common=program.keys()&snapshot.keys();conflicts=[];text_changes=[]
+    for forum in sorted(common):
+        p,s=program[forum],snapshot[forum]
+        if s['status']!='accepted':conflicts.append({'forum':forum,'official_program':'accepted','snapshot_status':s['status']})
+        same_title=normalized(p['title'])==normalized(s['title'])
+        same_abstract=(p.get('abstract') or '').strip()==(s.get('abstract') or '').strip()
+        overlap=len(author_keys(p.get('authors',[]))&author_keys(s.get('authors',[])))
+        if not same_title or not same_abstract:
+            text_changes.append({'forum':forum,'same_normalized_title':same_title,'same_abstract':same_abstract,'shared_public_authors':overlap})
+    statuses=Counter(row['status'] for row in snapshot.values())
+    if dict(statuses)!=sm.get('submission_status_counts'):raise ValueError('Snapshot status counts mismatch')
+    report={'year':year,'program_records':len(program),'snapshot_records':len(snapshot),'shared_forums':len(common),
+            'snapshot_status_counts':dict(statuses),'snapshot_missing_abstracts':sm['missing_abstract_count'],
+            'snapshot_observation_upper_bound':sm.get('snapshot_observation_upper_bound'),
+            'program_only_forums':sorted(program.keys()-snapshot.keys()),
+            'snapshot_accepted_not_in_program':sorted(forum for forum,row in snapshot.items() if row['status']=='accepted' and forum not in program),
+            'program_snapshot_status_differences':conflicts,'shared_forum_text_differences':text_changes,
+            'unmatched_public_decisions':sm.get('unmatched_public_decisions',0),
+            'interpretation':'Snapshot gaps and unknowns are not rejections. Strong forum identity links events; historical observations and text variants remain preserved.'}
+    return report,(str(program_path),ph),(str(snapshot_path),sh)
+
+
+def audit(root,years,report_path):
+    root=Path(root).resolve();reports=[];program={};snapshots={}
+    for year in sorted(set(years)):
+        report,p,s=audit_year(root,year);reports.append(report);program[p[0]]=p[1];snapshots[s[0]]=s[1]
+    result={'checked_at':timestamp(),'years':reports,'backfill_queue':{'files':list(program),'file_sha256':program},
+            'snapshot_files':{'files':list(snapshots),'file_sha256':snapshots}}
+    target=Path(report_path);part=target.with_suffix('.part')
+    part.write_text(json.dumps(result,ensure_ascii=False,indent=2));part.replace(target)
+    queue=target.with_name(target.stem+'-backfill-queue.json')
+    part=queue.with_suffix('.part');part.write_text(json.dumps(result['backfill_queue'],indent=2));part.replace(queue)
+    print(json.dumps({'report':str(target),'backfill_queue':str(queue),'years':[{'year':r['year'],'snapshot_status_counts':r['snapshot_status_counts'],'program_only':len(r['program_only_forums']),'status_differences':len(r['program_snapshot_status_differences'])} for r in reports]}),flush=True)
+    return result
+
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser();parser.add_argument('--root',required=True);parser.add_argument('--report',required=True)
+    parser.add_argument('--year',type=int,action='append',required=True);args=parser.parse_args()
+    audit(args.root,args.year,args.report)

test_iclr_snapshot_audit.py

@@ -0,0 +1,58 @@
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+from audit_iclr_snapshot import audit
+
+
+class SnapshotAuditTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup);self.root=Path(self.tmp.name)
+
+    def write(self,source,rows):
+        prefix='iclr-program' if source=='iclr_program' else 'openreview-snapshot'
+        path=self.root/f'{prefix}-2026-main.jsonl'
+        path.write_text(''.join(json.dumps({'source':source,'venue':'ICLR','year':2026,'track':'main',**row})+'\n' for row in rows))
+        manifest={'source':source,'venue':'ICLR','year':2026,'track':'main','parser_version':2,'count':len(rows),'missing_abstract_count':0}
+        if source=='openreview_snapshot':
+            manifest['submission_status_counts']={status:sum(r['status']==status for r in rows) for status in set(r['status'] for r in rows)}
+        path.with_suffix('.manifest.json').write_text(json.dumps(manifest));return path
+
+    def row(self,forum,status='accepted',number=None):
+        ids=[{'scheme':'openreview','value':forum}]
+        if number:ids.append({'scheme':'iclr_number','value':str(number)})
+        return {'external_id':forum,'identifiers':{'openreview':forum},'submission_identifiers':ids,
+                'title':'Title '+forum,'abstract':'Abstract','authors':['Jane Doe'],'status':status,
+                'metadata':{'public_readers_verified':True},'decision_events':[] if status=='unknown' else [
+                    {'status':status,'source_decision_id':'note-'+forum,'authority':{'public_readers_verified':True}}]}
+
+    def test_gaps_and_historical_status_differences_are_explicit_not_inferred(self):
+        self.write('iclr_program',[self.row('shared'),self.row('programOnly')])
+        self.write('openreview_snapshot',[self.row('shared','rejected',1),self.row('snapshotOnly','accepted',2),self.row('unknown','unknown',3)])
+        result=audit(self.root,[2026],self.root/'audit.json');report=result['years'][0]
+        self.assertEqual(report['program_only_forums'],['programOnly'])
+        self.assertEqual(report['snapshot_accepted_not_in_program'],['snapshotOnly'])
+        self.assertEqual(report['program_snapshot_status_differences'][0]['snapshot_status'],'rejected')
+        self.assertEqual(report['snapshot_status_counts']['rejected'],1)
+        self.assertTrue((self.root/'audit-backfill-queue.json').is_file())
+
+    def test_conflicting_numbers_fail_before_publishing_a_queue(self):
+        self.write('iclr_program',[self.row('one')])
+        self.write('openreview_snapshot',[self.row('one',number=1),self.row('two',number=1)])
+        with self.assertRaisesRegex(ValueError,'different forums'):audit(self.root,[2026],self.root/'audit.json')
+        self.assertFalse((self.root/'audit-backfill-queue.json').exists())
+
+    def test_missing_public_decision_evidence_is_rejected(self):
+        self.write('iclr_program',[self.row('one')])
+        row=self.row('one');row['decision_events'][0]['authority']={}
+        self.write('openreview_snapshot',[row])
+        with self.assertRaisesRegex(ValueError,'public decision authority'):audit(self.root,[2026],self.root/'audit.json')
+
+    def test_manifest_counts_cannot_silently_hide_missing_records(self):
+        path=self.write('iclr_program',[self.row('one')]);path.write_text('')
+        self.write('openreview_snapshot',[self.row('one')])
+        with self.assertRaisesRegex(ValueError,'count mismatch'):audit(self.root,[2026],self.root/'audit.json')
+
+
+if __name__=='__main__':unittest.main()