Extract public ICLR snapshot submissions and signed decision evidence
Metadata integrity checks / metadata (push) Failing after 5s

This commit is contained in:
2026-11-06 15:50:26 +08:00
parent 1213369824
commit 53a83a85a6
3 changed files with 321 additions and 2 deletions
+14 -2
View File
@@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。
**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序、CVF/ECVA 视觉会议元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的补充来源。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行,已配置 ces-control:host 对应的 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序、CVF/ECVA 视觉会议元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的历史补充;近期 ICLR 投稿与决定快照解析器正在验证。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),主批次已读至 EOF 并完成 3,233,620 条源记录的导入;这不是去重后的论文数,最终全库审计尚未完成。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行,已配置 ces-control:host 对应的 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
## 数据范围与来源
@@ -172,7 +172,19 @@ python3 -B download_snapshot.py \
--output /data/paper-index/raw/iclr-openreview-v1.0.tar.bz2
```
[快照来源与复现脚本](https://github.com/qhjqhj00/iclr-openreview-reviews)。原包包含评审回复;后续解析只筛选公开投稿的索引、标题、摘要、平台标签及带证据的明确决定,不提取评审正文、作者邮件或论文全文。必须核对原始 invitation、readers、签名及决定字段,不能根据评分或录用名单缺席推断拒稿。该解析适配尚在开发,下载成功不代表已经入库或核验覆盖。
[快照来源与复现脚本](https://github.com/qhjqhj00/iclr-openreview-reviews)。原包包含评审回复;解析只筛选公开投稿的索引、标题、摘要、平台标签及带证据的明确决定,不提取评审正文、作者邮件或论文全文。必须核对原始 invitation、readers、签名及决定字段,不能根据评分或录用名单缺席推断拒稿。
优先处理近期三届,再单独补全 2020–2023;不以 2017 年的 PeerRead 样本代表近期拒稿覆盖:
```sh
python3 -B collect_openreview_snapshot.py \
--archive /data/paper-index/raw/iclr-openreview-v1.0.tar.bz2 \
--out /data/paper-index/collected --year 2024 --year 2025 --year 2026
```
解析器先核验固定 SHA-256,再流式读取 tar,以临时 SQLite 按年份和 forum 关联投稿及决定,兼容回复先于投稿的归档顺序。只接受明确对 everyone 公开的投稿和具有会议主席签名的 Decision;保留原始决定值、note ID、决定日期、invitation 和签名证据。无明确决定的投稿保留 unknown,不产生拒稿推断。无法关联到公开投稿元数据的公开决定另存 unmatched-decisions 文件,供复核,不擅自创建论文。输出清单分别统计投稿状态、决定事件、缺失摘要和排除原因;重复副本去重,内容冲突会中止。
该版本是 2026-05-08 发布的第三方历史副本,不代表平台当前状态或全部投稿;缺失决定及公开性筛选均需核验。采集成功也不等于主库已更新,后续应通过单写入协调器导入,并核对同届官方程序与公开决定的差异。
下载异常后先检查对应进程或 CI 任务确已终止,再使用同一参数增加 `--resume`。只有断点身份一致且服务器实际返回正确 Range 响应才续传;不支持时保留断点并报错,不自动重下。正式文件与下载清单均放在仓库外,不提交大型来源包。
+200
View File
@@ -0,0 +1,200 @@
"""Extract public paper metadata and signed Decision notes from a pinned release."""
import argparse
from collections import Counter,defaultdict
from contextlib import closing
from datetime import datetime,timezone
import json
from pathlib import Path
import re
import sqlite3
import tarfile
import tempfile
from download_snapshot import digest_file
SOURCE='openreview_snapshot'
SHA256='350071daf25ebf1aef854ee1d5dc3a81834e1c468917c82a4f78af5548de34fc'
RELEASE='https://github.com/qhjqhj00/iclr-openreview-reviews/releases/tag/v1.0'
AS_OF='2026-05-08T04:50:23.000Z'
EXPECTED={2020:2213,2021:2594,2022:2617,2023:3792,2024:7404,2025:11672,2026:19814}
def value(content,key,default=None):
result=content.get(key,default)
return result.get('value',default) if isinstance(result,dict) else result
def invitations(note):
result=note.get('invitations') or []
if isinstance(result,str):result=[result]
if note.get('invitation'):result=list(result)+[note['invitation']]
return [i for i in result if isinstance(i,str)]
def public(note):
return isinstance(note.get('readers'),list) and 'everyone' in note['readers']
def date_ms(raw):
if raw is None:return None
return datetime.fromtimestamp(int(raw)/1000,timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z')
def decision_status(raw):
key=' '.join(raw.casefold().replace('_',' ').split())
if key.startswith(('desk reject','desk-reject')):return 'desk_rejected'
if key in ('accept','accepted','poster','oral','spotlight') or key.startswith(('accept (','accept:','accept -')):return 'accepted'
if key in ('reject','rejected','reject and resubmit') or key.startswith(('reject (','reject:','reject -')):return 'rejected'
if key in ('withdraw','withdrawn','withdrawal'):return 'withdrawn'
return 'unknown'
def paper_record(note,year):
if not public(note):return None,'not_public'
base=f'ICLR.cc/{year}/Conference/-/'
if not any(i in (base+'Submission',base+'Blind_Submission') for i in invitations(note)):return None,'wrong_invitation'
forum=note.get('id')
if not isinstance(forum,str) or not re.fullmatch(r'[A-Za-z0-9_-]+',forum) or note.get('forum') not in (None,forum):return None,'not_submission_root'
content=note.get('content') or {};title=value(content,'title');abstract=value(content,'abstract')
if not isinstance(title,str) or not title.strip():return None,'missing_title'
if abstract is not None and not isinstance(abstract,str):return None,'nontext_abstract'
authors=value(content,'authors',[]) or []
if isinstance(authors,str):authors=[a.strip() for a in authors.split(',')]
if not isinstance(authors,list):return None,'invalid_authors'
authors=[a.strip() for a in authors if isinstance(a,str) and a.strip() and not re.fullmatch(r'anonymous(?: authors?)?',a.strip(),re.I)]
number=note.get('number');number=int(number) if re.fullmatch(r'[1-9]\d*',str(number)) else None
identifiers={'openreview':forum}
for field in ('doi','arxiv','arxiv_id'):
own=value(content,field)
if not isinstance(own,str):continue
own=own.strip()
if field=='doi':
own=re.sub(r'^https?://(?:dx\.)?doi\.org/','',own,flags=re.I).lower()
if re.fullmatch(r'10\.\d{4,9}/\S+',own):identifiers['doi']=own
else:
own=re.sub(r'^https?://arxiv.org/(?:abs|pdf)/','',own).removesuffix('.pdf')
if re.fullmatch(r'(?:\d{4}\.\d{4,5}|[a-zA-Z.-]+/\d{7})(?:v\d+)?',own):identifiers['arxiv']=own
tags=[];index_fields={}
for field in ('keywords','primary_area','secondary_area','Please_choose_the_closest_area_that_your_submission_falls_into'):
raw=value(content,field)
if raw is None:continue
index_fields[field]=raw
terms=raw if isinstance(raw,list) else re.split(r'[,;\n]',raw) if field=='keywords' and isinstance(raw,str) else [raw]
namespace=SOURCE+':'+('keyword' if field=='keywords' else field)
tags.extend({'namespace':namespace,'value':term.strip()} for term in terms if isinstance(term,str) and term.strip())
ids=[{'scheme':'openreview','value':forum}]
if number:ids.append({'scheme':'iclr_number','value':str(number)})
return {'source':SOURCE,'external_id':f'{year}/{forum}','url':'https://openreview.net/forum?id='+forum,
'title':' '.join(title.split()),'abstract':abstract.strip() if abstract else None,'authors':authors,
'identifiers':identifiers,'submission_identifiers':ids,'venue':'ICLR','year':year,'track':'main',
'submitted_at':date_ms(note.get('cdate') or note.get('tcdate')),'observed_at':AS_OF,'tags':tags,
'metadata':{'forum':forum,'source_submission_number':number,'original_invitations':invitations(note),
'public_readers_verified':True,'index_fields':index_fields,'venue':value(content,'venue'),
'venueid':value(content,'venueid'),'snapshot_release':RELEASE,
'snapshot_observation_upper_bound':AS_OF}},None
def decision_record(note,year):
if not public(note):return None,'not_public'
base=f'ICLR.cc/{year}/Conference/'
if not any(i.startswith(base) and i.endswith('/Decision') for i in invitations(note)):return None,'not_decision'
if not any(isinstance(s,str) and s.startswith(base) and s.rsplit('/',1)[-1] in ('Program_Chairs','Area_Chairs','Senior_Area_Chairs') for s in note.get('signatures',[])):
return None,'unverified_signature'
if note.get('ddate'):return None,'deleted_decision'
forum=note.get('forum');identifier=note.get('id')
if not all(isinstance(v,str) and re.fullmatch(r'[A-Za-z0-9_-]+',v) for v in (forum,identifier)):return None,'invalid_identity'
if note.get('_paper_forum',forum)!=forum:return None,'forum_mismatch'
raw=value(note.get('content') or {},'decision')
if not isinstance(raw,str) or not raw.strip():return None,'missing_decision_value'
return {'id':identifier,'forum':forum,'status':decision_status(raw),'raw_value':raw,
'evidence_url':f'https://openreview.net/forum?id={forum}&noteId={identifier}',
'observed_at':AS_OF,'decision_date':date_ms(note.get('cdate') or note.get('tcdate') or note.get('mdate')),
'source_decision_id':identifier,'authority':{'invitations':invitations(note),
'signatures':note.get('signatures',[]),'public_readers_verified':True}},None
def collect(archive,out,years,expected_sha=SHA256):
actual=digest_file(Path(archive))
if actual['sha256']!=expected_sha:raise ValueError('Snapshot checksum mismatch before parsing')
if expected_sha==SHA256 and actual['bytes']!=512940722:raise ValueError('Unexpected pinned snapshot size')
out=Path(out);out.mkdir(parents=True,exist_ok=True);years=set(years)
counts=defaultdict(Counter);members=set()
with tempfile.TemporaryDirectory(prefix='openreview-stage-',dir=out) as tmp,closing(sqlite3.connect(Path(tmp)/'metadata.sqlite')) as c:
c.executescript('PRAGMA synchronous=OFF; CREATE TABLE papers(year INTEGER,forum TEXT,payload TEXT,PRIMARY KEY(year,forum)); CREATE TABLE decisions(year INTEGER,id TEXT,forum TEXT,payload TEXT,PRIMARY KEY(year,id)); CREATE INDEX decisions_forum ON decisions(year,forum);')
with tarfile.open(archive,mode='r|bz2') as tar:
for member in tar:
match=re.fullmatch(r'(?:\./)?data/iclr_(20\d{2})/(papers|reviews)\.jsonl',member.name)
if not match or int(match[1]) not in years:continue
year=int(match[1]);kind=match[2]
if not member.isfile() or (year,kind) in members:raise ValueError('Invalid or repeated metadata member')
members.add((year,kind));lines=0
with tar.extractfile(member) as stream:
while line:=stream.readline(16*1024**2+1):
if len(line)>16*1024**2:raise ValueError('Unexpectedly large metadata line')
if not line.strip():continue
lines+=1;counts[year][kind+'_lines']+=1
if kind=='reviews' and b'/Decision' not in line:continue
note=json.loads(line)
row,reason=paper_record(note,year) if kind=='papers' else decision_record(note,year)
if row is None:
counts[year][kind+'_excluded_'+reason]+=1;continue
if kind=='papers':
forum=row['metadata']['forum'];payload=json.dumps(row,ensure_ascii=False)
old=c.execute('SELECT payload FROM papers WHERE year=? AND forum=?',(year,forum)).fetchone()
if old and old[0]!=payload:raise ValueError('Conflicting copies of one submission')
if old:counts[year]['duplicate_papers']+=1
else:c.execute('INSERT INTO papers VALUES(?,?,?)',(year,forum,payload))
else:
payload=json.dumps(row,ensure_ascii=False);old=c.execute('SELECT payload FROM decisions WHERE year=? AND id=?',(year,row['id'])).fetchone()
if old and old[0]!=payload:raise ValueError('Conflicting copies of one decision note')
if old:counts[year]['duplicate_decisions']+=1
else:c.execute('INSERT INTO decisions VALUES(?,?,?,?)',(year,row['id'],row['forum'],payload))
if lines%5000==0:print(json.dumps({'source':SOURCE,'year':year,'member':kind,'lines':lines}),flush=True)
c.commit();print(json.dumps({'source':SOURCE,'year':year,'member':kind,'lines':lines,'status':'member_processed'}),flush=True)
if members!={(year,kind) for year in years for kind in ('papers','reviews')}:raise ValueError('Selected year metadata members are missing')
if expected_sha==SHA256:
for year in years:
if counts[year]['papers_lines']!=EXPECTED.get(year):raise ValueError('Pinned release paper count differs from published inventory')
status_path=out/'openreview-snapshot-year-status.json';catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
for year in sorted(years,reverse=True):
target=out/f'openreview-snapshot-{year}-main.jsonl';part=target.with_suffix('.part')
statuses=Counter();event_counts=Counter();missing=0;written=0
with part.open('w') as output:
for forum,payload in c.execute('SELECT forum,payload FROM papers WHERE year=? ORDER BY forum',(year,)):
row=json.loads(payload)
events=[json.loads(r[0]) for r in c.execute('SELECT payload FROM decisions WHERE year=? AND forum=?',(year,forum))]
events.sort(key=lambda event:(event['decision_date'] or '',event['id']))
row['decision_events']=events;row['status']=events[-1]['status'] if events else 'unknown'
row['metadata']['snapshot_sha256']=actual['sha256'];row['metadata']['decision_note_ids']=[e['id'] for e in events]
row['metadata']['decision_authority']=[{'note_id':e['id'],**e['authority']} for e in events]
output.write(json.dumps(row,ensure_ascii=False)+'\n');written+=1;missing+=not bool(row['abstract']);statuses[row['status']]+=1
event_counts.update(e['status'] for e in events)
if not written:raise ValueError('No verified public submissions for selected year')
unmatched=c.execute('SELECT count(*) FROM decisions d WHERE d.year=? AND NOT EXISTS(SELECT 1 FROM papers p WHERE p.year=d.year AND p.forum=d.forum)',(year,)).fetchone()[0]
unmatched_path=None
if unmatched:
unresolved=out/f'openreview-snapshot-{year}-unmatched-decisions.jsonl'
with unresolved.with_suffix('.part').open('w') as output:
for payload, in c.execute('SELECT payload FROM decisions d WHERE d.year=? AND NOT EXISTS(SELECT 1 FROM papers p WHERE p.year=d.year AND p.forum=d.forum)',(year,)):
event=json.loads(payload);event.update(year=year,reason='public decision lacks public submission metadata in snapshot',snapshot_sha256=actual['sha256'])
output.write(json.dumps(event,ensure_ascii=False)+'\n')
unresolved.with_suffix('.part').replace(unresolved);unmatched_path=str(unresolved)
part.replace(target)
manifest={'source':SOURCE,'parser_version':1,'venue':'ICLR','year':year,'track':'main','count':written,
'list_url':RELEASE,'snapshot_sha256':actual['sha256'],'snapshot_observation_upper_bound':AS_OF,
'missing_abstract_count':missing,'submission_status_counts':dict(statuses),'decision_event_counts':dict(event_counts),
'unmatched_public_decisions':unmatched,'archive_counts':dict(counts[year]),'file':str(target),
'unmatched_evidence_file':unmatched_path,
'status':'partial' if missing or statuses['unknown'] or unmatched or any('_excluded_' in k for k in counts[year]) else 'collected'}
target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));catalogue[str(year)]=manifest
print(json.dumps(manifest),flush=True)
stage=status_path.with_suffix('.part');stage.write_text(json.dumps(catalogue,indent=2));stage.replace(status_path)
(out/'openreview-snapshot-last-run.json').write_text(json.dumps({'requested_years':sorted(years),'sha256':actual['sha256'],'release':RELEASE,'observed_at_upper_bound':AS_OF},indent=2))
return {year:catalogue[str(year)] for year in years}
if __name__=='__main__':
parser=argparse.ArgumentParser();parser.add_argument('--archive',required=True);parser.add_argument('--out',required=True)
parser.add_argument('--year',type=int,action='append');args=parser.parse_args()
collect(args.archive,args.out,args.year or sorted(EXPECTED))
+107
View File
@@ -0,0 +1,107 @@
import hashlib
import io
import json
from pathlib import Path
import sqlite3
import tarfile
import tempfile
import unittest
from collect_openreview_snapshot import collect,decision_record,paper_record
from import_records import ingest_file
class SnapshotTests(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
self.root=Path(self.tmp.name)
def paper(self,forum='paper1',v2=True):
content={'title':'A public paper','abstract':'A public abstract','authors':['Jane Doe'],
'authorids':['private@example.org'],'keywords':['learning']}
if v2:content={key:{'value':value} for key,value in content.items()}
return {'id':forum,'forum':forum,'number':42,'readers':['everyone'],'cdate':1760000000000,
'invitations':['ICLR.cc/2026/Conference/-/Submission'],'content':content}
def decision(self,forum='paper1',identifier='decision1',value='Reject'):
return {'id':identifier,'forum':forum,'_paper_forum':forum,'readers':['everyone'],
'invitations':['ICLR.cc/2026/Conference/Submission42/-/Decision'],
'signatures':['ICLR.cc/2026/Conference/Program_Chairs'],'cdate':1770000000000,
'content':{'decision':{'value':value},'comment':{'value':'PRIVATE REVIEW BODY'}}}
def archive(self,papers,reviews):
path=self.root/'snapshot.tar.bz2'
with tarfile.open(path,'w:bz2') as archive:
# Real release order is not guaranteed; decisions precede submissions.
for kind,rows in [('reviews',reviews),('papers',papers)]:
data=(''.join(json.dumps(row)+'\n' for row in rows)).encode()
entry=tarfile.TarInfo('data/iclr_2026/'+kind+'.jsonl');entry.size=len(data)
archive.addfile(entry,io.BytesIO(data))
return path,hashlib.sha256(path.read_bytes()).hexdigest()
def test_public_v1_and_v2_metadata_omit_authorids_and_review_bodies(self):
for v2 in (False,True):
row,reason=paper_record(self.paper(v2=v2),2026)
self.assertIsNone(reason);self.assertEqual(row['authors'],['Jane Doe'])
self.assertNotIn('private@example.org',json.dumps(row))
self.assertEqual(row['identifiers']['openreview'],'paper1')
note=self.paper();note['readers']=['chairs']
self.assertEqual(paper_record(note,2026)[1],'not_public')
def test_only_public_signed_formal_decisions_count(self):
note=self.decision();row,reason=decision_record(note,2026)
self.assertIsNone(reason);self.assertEqual(row['status'],'rejected')
self.assertNotIn('PRIVATE REVIEW BODY',json.dumps(row))
note['invitations']=['ICLR.cc/2026/Conference/Submission42/-/Official_Review']
self.assertEqual(decision_record(note,2026)[1],'not_decision')
note=self.decision();note['signatures']=['ICLR.cc/2026/Conference/Submission42/Reviewer_1']
self.assertEqual(decision_record(note,2026)[1],'unverified_signature')
note=self.decision();note['readers']=['chairs']
self.assertEqual(decision_record(note,2026)[1],'not_public')
note=self.decision(value='Borderline Accept')
self.assertEqual(decision_record(note,2026)[0]['status'],'unknown')
def test_unordered_snapshot_import_is_idempotent_and_preserves_unknown_and_orphans(self):
path,sha=self.archive([self.paper(),self.paper('paper2'),self.paper('paper3')],
[self.decision(),self.decision('paper2','decision2','Accept (poster)'),self.decision('absent','orphan')])
out=self.root/'out';manifest=collect(path,out,[2026],sha)[2026]
self.assertEqual(manifest['submission_status_counts'],{'rejected':1,'accepted':1,'unknown':1})
self.assertEqual(manifest['unmatched_public_decisions'],1)
self.assertEqual(len(Path(manifest['unmatched_evidence_file']).read_text().splitlines()),1)
rows=[json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
self.assertEqual(rows[2]['decision_events'],[])
self.assertNotIn('PRIVATE REVIEW BODY',json.dumps(rows))
db=self.root/'db.sqlite'
ingest_file(db,manifest['file']);ingest_file(db,manifest['file'])
with sqlite3.connect(db) as connection:
self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0],3)
self.assertEqual(connection.execute('SELECT count(*) FROM decisions').fetchone()[0],2)
self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_identical_copies_coalesce_and_conflicting_copies_fail(self):
paper=self.paper();decision=self.decision()
path,sha=self.archive([paper,paper],[decision,decision])
manifest=collect(path,self.root/'out',[2026],sha)[2026]
self.assertEqual(manifest['count'],1)
self.assertEqual(manifest['archive_counts']['duplicate_decisions'],1)
different=self.paper();different['content']['title']={'value':'Conflicting title'}
path,sha=self.archive([paper,different],[decision])
with self.assertRaisesRegex(ValueError,'Conflicting copies'):
collect(path,self.root/'bad',[2026],sha)
self.assertFalse((self.root/'bad'/'openreview-snapshot-2026-main.jsonl').exists())
def test_checksum_failure_does_not_publish_output(self):
path,sha=self.archive([self.paper()],[self.decision()])
with self.assertRaisesRegex(ValueError,'checksum mismatch'):
collect(path,self.root/'out',[2026],'0'*64)
self.assertFalse((self.root/'out').exists())
def test_legacy_invitation_and_plain_decision_value(self):
note=self.decision();note['invitation']=note.pop('invitations')[0]
note['content']['decision']='Desk Reject'
self.assertEqual(decision_record(note,2026)[0]['status'],'desk_rejected')
note['_paper_forum']='wrong'
self.assertEqual(decision_record(note,2026)[1],'forum_mismatch')
if __name__=='__main__':unittest.main()