Files
paper-atlas/paper_atlas/collectors/openreview.py
codex-bot 1c18cbe03a
Metadata integrity checks / metadata (push) Successful in 1m12s
Package Paper Atlas with organized modules and a unified CLI
2026-11-06 17:32:24 +08:00

205 lines
14 KiB
Python

"""Extract public paper metadata and signed Decision notes from a pinned release."""
import argparse
from collections import Counter,defaultdict
from contextlib import closing
from datetime import datetime,timezone
import json
from pathlib import Path
import re
import sqlite3
import tarfile
import tempfile
from paper_atlas.io.snapshots import digest_file
SOURCE='openreview_snapshot'
SHA256='350071daf25ebf1aef854ee1d5dc3a81834e1c468917c82a4f78af5548de34fc'
RELEASE='https://github.com/qhjqhj00/iclr-openreview-reviews/releases/tag/v1.0'
AS_OF='2026-05-08T04:50:23.000Z'
EXPECTED={2020:2213,2021:2594,2022:2617,2023:3792,2024:7404,2025:11672,2026:19814}
def value(content,key,default=None):
result=content.get(key,default)
return result.get('value',default) if isinstance(result,dict) else result
def invitations(note):
result=note.get('invitations') or []
if isinstance(result,str):result=[result]
if note.get('invitation'):result=list(result)+[note['invitation']]
return [i for i in result if isinstance(i,str)]
def public(note):
return isinstance(note.get('readers'),list) and 'everyone' in note['readers']
def date_ms(raw):
if raw is None:return None
return datetime.fromtimestamp(int(raw)/1000,timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z')
def decision_status(raw):
key=' '.join(raw.casefold().replace('_',' ').split())
if key.startswith(('desk reject','desk-reject')):return 'desk_rejected'
if key in ('accept','accepted','poster','oral','spotlight') or key.startswith(('accept (','accept:','accept -')):return 'accepted'
if key in ('reject','rejected','reject and resubmit') or key.startswith(('reject (','reject:','reject -')):return 'rejected'
if key in ('withdraw','withdrawn','withdrawal'):return 'withdrawn'
return 'unknown'
def paper_record(note,year):
if not public(note):return None,'not_public'
base=f'ICLR.cc/{year}/Conference/-/'
if not any(i in (base+'Submission',base+'Blind_Submission') for i in invitations(note)):return None,'wrong_invitation'
forum=note.get('id')
if not isinstance(forum,str) or not re.fullmatch(r'[A-Za-z0-9_-]+',forum) or note.get('forum') not in (None,forum):return None,'not_submission_root'
content=note.get('content') or {};title=value(content,'title');abstract=value(content,'abstract')
if not isinstance(title,str) or not title.strip():return None,'missing_title'
if abstract is not None and not isinstance(abstract,str):return None,'nontext_abstract'
authors=value(content,'authors',[]) or []
if isinstance(authors,str):authors=[a.strip() for a in authors.split(',')]
if not isinstance(authors,list):return None,'invalid_authors'
authors=[a.strip() for a in authors if isinstance(a,str) and a.strip() and not re.fullmatch(r'anonymous(?: authors?)?',a.strip(),re.I)]
number=note.get('number');number=int(number) if re.fullmatch(r'[1-9]\d*',str(number)) else None
identifiers={'openreview':forum}
for field in ('doi','arxiv','arxiv_id'):
own=value(content,field)
if not isinstance(own,str):continue
own=own.strip()
if field=='doi':
own=re.sub(r'^https?://(?:dx\.)?doi\.org/','',own,flags=re.I).lower()
if re.fullmatch(r'10\.\d{4,9}/\S+',own):identifiers['doi']=own
else:
own=re.sub(r'^https?://arxiv.org/(?:abs|pdf)/','',own).removesuffix('.pdf')
if re.fullmatch(r'(?:\d{4}\.\d{4,5}|[a-zA-Z.-]+/\d{7})(?:v\d+)?',own):identifiers['arxiv']=own
tags=[];index_fields={}
for field in ('keywords','primary_area','secondary_area','Please_choose_the_closest_area_that_your_submission_falls_into'):
raw=value(content,field)
if raw is None:continue
index_fields[field]=raw
terms=raw if isinstance(raw,list) else re.split(r'[,;\n]',raw) if field=='keywords' and isinstance(raw,str) else [raw]
namespace=SOURCE+':'+('keyword' if field=='keywords' else field)
tags.extend({'namespace':namespace,'value':term.strip()} for term in terms if isinstance(term,str) and term.strip())
ids=[{'scheme':'openreview','value':forum}]
if number:ids.append({'scheme':'iclr_number','value':str(number)})
return {'source':SOURCE,'external_id':f'{year}/{forum}','url':'https://openreview.net/forum?id='+forum,
'title':' '.join(title.split()),'abstract':abstract.strip() if abstract else None,'authors':authors,
'identifiers':identifiers,'submission_identifiers':ids,'venue':'ICLR','year':year,'track':'main',
'submitted_at':date_ms(note.get('cdate') or note.get('tcdate')),'observed_at':AS_OF,'tags':tags,
'metadata':{'forum':forum,'source_submission_number':number,'original_invitations':invitations(note),
'public_readers_verified':True,'index_fields':index_fields,'venue':value(content,'venue'),
'venueid':value(content,'venueid'),'snapshot_release':RELEASE,
'snapshot_observation_upper_bound':AS_OF}},None
def decision_record(note,year):
if not public(note):return None,'not_public'
base=f'ICLR.cc/{year}/Conference/'
if not any(i.startswith(base) and i.endswith('/Decision') for i in invitations(note)):return None,'not_decision'
if not any(isinstance(s,str) and s.startswith(base) and s.rsplit('/',1)[-1] in ('Program_Chairs','Area_Chairs','Senior_Area_Chairs') for s in note.get('signatures',[])):
return None,'unverified_signature'
if note.get('ddate'):return None,'deleted_decision'
forum=note.get('forum');identifier=note.get('id')
if not all(isinstance(v,str) and re.fullmatch(r'[A-Za-z0-9_-]+',v) for v in (forum,identifier)):return None,'invalid_identity'
if note.get('_paper_forum',forum)!=forum:return None,'forum_mismatch'
raw=value(note.get('content') or {},'decision')
if not isinstance(raw,str) or not raw.strip():return None,'missing_decision_value'
return {'id':identifier,'forum':forum,'status':decision_status(raw),'raw_value':raw,
'evidence_url':f'https://openreview.net/forum?id={forum}&noteId={identifier}',
'observed_at':AS_OF,'decision_date':date_ms(note.get('cdate') or note.get('tcdate') or note.get('mdate')),
'source_decision_id':identifier,'authority':{'invitations':invitations(note),
'signatures':note.get('signatures',[]),'public_readers_verified':True}},None
def collect(archive,out,years,expected_sha=SHA256):
actual=digest_file(Path(archive))
if actual['sha256']!=expected_sha:raise ValueError('Snapshot checksum mismatch before parsing')
if expected_sha==SHA256 and actual['bytes']!=512940722:raise ValueError('Unexpected pinned snapshot size')
out=Path(out);out.mkdir(parents=True,exist_ok=True);years=set(years)
counts=defaultdict(Counter);members=set()
with tempfile.TemporaryDirectory(prefix='openreview-stage-',dir=out) as tmp,closing(sqlite3.connect(Path(tmp)/'metadata.sqlite')) as c:
c.executescript('PRAGMA synchronous=OFF; CREATE TABLE papers(year INTEGER,forum TEXT,payload TEXT,PRIMARY KEY(year,forum)); CREATE TABLE decisions(year INTEGER,id TEXT,forum TEXT,payload TEXT,PRIMARY KEY(year,id)); CREATE INDEX decisions_forum ON decisions(year,forum);')
with tarfile.open(archive,mode='r|bz2') as tar:
for member in tar:
match=re.fullmatch(r'(?:\./)?data/iclr_(20\d{2})/(papers|reviews)\.jsonl',member.name)
if not match or int(match[1]) not in years:continue
year=int(match[1]);kind=match[2]
if not member.isfile() or (year,kind) in members:raise ValueError('Invalid or repeated metadata member')
members.add((year,kind));lines=0
with tar.extractfile(member) as stream:
while line:=stream.readline(16*1024**2+1):
if len(line)>16*1024**2:raise ValueError('Unexpectedly large metadata line')
if not line.strip():continue
lines+=1;counts[year][kind+'_lines']+=1
if kind=='reviews' and b'/Decision' not in line:continue
note=json.loads(line)
row,reason=paper_record(note,year) if kind=='papers' else decision_record(note,year)
if row is None:
counts[year][kind+'_excluded_'+reason]+=1;continue
if kind=='papers':
forum=row['metadata']['forum'];payload=json.dumps(row,ensure_ascii=False)
old=c.execute('SELECT payload FROM papers WHERE year=? AND forum=?',(year,forum)).fetchone()
if old and old[0]!=payload:raise ValueError('Conflicting copies of one submission')
if old:counts[year]['duplicate_papers']+=1
else:c.execute('INSERT INTO papers VALUES(?,?,?)',(year,forum,payload))
else:
payload=json.dumps(row,ensure_ascii=False);old=c.execute('SELECT payload FROM decisions WHERE year=? AND id=?',(year,row['id'])).fetchone()
if old and old[0]!=payload:raise ValueError('Conflicting copies of one decision note')
if old:counts[year]['duplicate_decisions']+=1
else:c.execute('INSERT INTO decisions VALUES(?,?,?,?)',(year,row['id'],row['forum'],payload))
if lines%5000==0:print(json.dumps({'source':SOURCE,'year':year,'member':kind,'lines':lines}),flush=True)
c.commit();print(json.dumps({'source':SOURCE,'year':year,'member':kind,'lines':lines,'status':'member_processed'}),flush=True)
if members!={(year,kind) for year in years for kind in ('papers','reviews')}:raise ValueError('Selected year metadata members are missing')
if expected_sha==SHA256:
for year in years:
if counts[year]['papers_lines']!=EXPECTED.get(year):raise ValueError('Pinned release paper count differs from published inventory')
status_path=out/'openreview-snapshot-year-status.json';catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
for year in sorted(years,reverse=True):
target=out/f'openreview-snapshot-{year}-main.jsonl';part=target.with_suffix('.part')
statuses=Counter();event_counts=Counter();missing=0;written=0
with part.open('w') as output:
for forum,payload in c.execute('SELECT forum,payload FROM papers WHERE year=? ORDER BY forum',(year,)):
row=json.loads(payload)
events=[json.loads(r[0]) for r in c.execute('SELECT payload FROM decisions WHERE year=? AND forum=?',(year,forum))]
events.sort(key=lambda event:(event['decision_date'] or '',event['id']))
row['decision_events']=events;row['status']=events[-1]['status'] if events else 'unknown'
row['metadata']['snapshot_sha256']=actual['sha256'];row['metadata']['decision_note_ids']=[e['id'] for e in events]
row['metadata']['decision_authority']=[{'note_id':e['id'],**e['authority']} for e in events]
output.write(json.dumps(row,ensure_ascii=False)+'\n');written+=1;missing+=not bool(row['abstract']);statuses[row['status']]+=1
event_counts.update(e['status'] for e in events)
if not written:raise ValueError('No verified public submissions for selected year')
unmatched=c.execute('SELECT count(*) FROM decisions d WHERE d.year=? AND NOT EXISTS(SELECT 1 FROM papers p WHERE p.year=d.year AND p.forum=d.forum)',(year,)).fetchone()[0]
unmatched_path=None
if unmatched:
unresolved=out/f'openreview-snapshot-{year}-unmatched-decisions.jsonl'
with unresolved.with_suffix('.part').open('w') as output:
for payload, in c.execute('SELECT payload FROM decisions d WHERE d.year=? AND NOT EXISTS(SELECT 1 FROM papers p WHERE p.year=d.year AND p.forum=d.forum)',(year,)):
event=json.loads(payload);event.update(year=year,reason='public decision lacks public submission metadata in snapshot',snapshot_sha256=actual['sha256'])
output.write(json.dumps(event,ensure_ascii=False)+'\n')
unresolved.with_suffix('.part').replace(unresolved);unmatched_path=str(unresolved)
part.replace(target)
manifest={'source':SOURCE,'parser_version':1,'venue':'ICLR','year':year,'track':'main','count':written,
'list_url':RELEASE,'snapshot_sha256':actual['sha256'],'snapshot_observation_upper_bound':AS_OF,
'missing_abstract_count':missing,'submission_status_counts':dict(statuses),'decision_event_counts':dict(event_counts),
'unmatched_public_decisions':unmatched,'archive_counts':dict(counts[year]),'file':str(target),
'unmatched_evidence_file':unmatched_path,
'status':'partial' if missing or statuses['unknown'] or unmatched or any('_excluded_' in k for k in counts[year]) else 'collected'}
target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));catalogue[str(year)]=manifest
print(json.dumps(manifest),flush=True)
stage=status_path.with_suffix('.part');stage.write_text(json.dumps(catalogue,indent=2));stage.replace(status_path)
(out/'openreview-snapshot-last-run.json').write_text(json.dumps({'requested_years':sorted(years),'sha256':actual['sha256'],'release':RELEASE,'observed_at_upper_bound':AS_OF},indent=2))
return {year:catalogue[str(year)] for year in years}
def main(argv=None, *, prog=None):
parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--archive',required=True);parser.add_argument('--out',required=True)
parser.add_argument('--year',type=int,action='append');args=parser.parse_args(argv)
collect(args.archive,args.out,args.year or sorted(EXPECTED))
if __name__ == '__main__':
main()