205 lines
14 KiB
Python
205 lines
14 KiB
Python
"""Extract public paper metadata and signed Decision notes from a pinned release."""
|
|
import argparse
|
|
from collections import Counter,defaultdict
|
|
from contextlib import closing
|
|
from datetime import datetime,timezone
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sqlite3
|
|
import tarfile
|
|
import tempfile
|
|
|
|
from paper_atlas.io.snapshots import digest_file
|
|
|
|
|
|
SOURCE='openreview_snapshot'
|
|
SHA256='350071daf25ebf1aef854ee1d5dc3a81834e1c468917c82a4f78af5548de34fc'
|
|
RELEASE='https://github.com/qhjqhj00/iclr-openreview-reviews/releases/tag/v1.0'
|
|
AS_OF='2026-05-08T04:50:23.000Z'
|
|
EXPECTED={2020:2213,2021:2594,2022:2617,2023:3792,2024:7404,2025:11672,2026:19814}
|
|
|
|
|
|
def value(content,key,default=None):
|
|
result=content.get(key,default)
|
|
return result.get('value',default) if isinstance(result,dict) else result
|
|
|
|
|
|
def invitations(note):
|
|
result=note.get('invitations') or []
|
|
if isinstance(result,str):result=[result]
|
|
if note.get('invitation'):result=list(result)+[note['invitation']]
|
|
return [i for i in result if isinstance(i,str)]
|
|
|
|
|
|
def public(note):
|
|
return isinstance(note.get('readers'),list) and 'everyone' in note['readers']
|
|
|
|
|
|
def date_ms(raw):
|
|
if raw is None:return None
|
|
return datetime.fromtimestamp(int(raw)/1000,timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z')
|
|
|
|
|
|
def decision_status(raw):
|
|
key=' '.join(raw.casefold().replace('_',' ').split())
|
|
if key.startswith(('desk reject','desk-reject')):return 'desk_rejected'
|
|
if key in ('accept','accepted','poster','oral','spotlight') or key.startswith(('accept (','accept:','accept -')):return 'accepted'
|
|
if key in ('reject','rejected','reject and resubmit') or key.startswith(('reject (','reject:','reject -')):return 'rejected'
|
|
if key in ('withdraw','withdrawn','withdrawal'):return 'withdrawn'
|
|
return 'unknown'
|
|
|
|
|
|
def paper_record(note,year):
|
|
if not public(note):return None,'not_public'
|
|
base=f'ICLR.cc/{year}/Conference/-/'
|
|
if not any(i in (base+'Submission',base+'Blind_Submission') for i in invitations(note)):return None,'wrong_invitation'
|
|
forum=note.get('id')
|
|
if not isinstance(forum,str) or not re.fullmatch(r'[A-Za-z0-9_-]+',forum) or note.get('forum') not in (None,forum):return None,'not_submission_root'
|
|
content=note.get('content') or {};title=value(content,'title');abstract=value(content,'abstract')
|
|
if not isinstance(title,str) or not title.strip():return None,'missing_title'
|
|
if abstract is not None and not isinstance(abstract,str):return None,'nontext_abstract'
|
|
authors=value(content,'authors',[]) or []
|
|
if isinstance(authors,str):authors=[a.strip() for a in authors.split(',')]
|
|
if not isinstance(authors,list):return None,'invalid_authors'
|
|
authors=[a.strip() for a in authors if isinstance(a,str) and a.strip() and not re.fullmatch(r'anonymous(?: authors?)?',a.strip(),re.I)]
|
|
number=note.get('number');number=int(number) if re.fullmatch(r'[1-9]\d*',str(number)) else None
|
|
identifiers={'openreview':forum}
|
|
for field in ('doi','arxiv','arxiv_id'):
|
|
own=value(content,field)
|
|
if not isinstance(own,str):continue
|
|
own=own.strip()
|
|
if field=='doi':
|
|
own=re.sub(r'^https?://(?:dx\.)?doi\.org/','',own,flags=re.I).lower()
|
|
if re.fullmatch(r'10\.\d{4,9}/\S+',own):identifiers['doi']=own
|
|
else:
|
|
own=re.sub(r'^https?://arxiv.org/(?:abs|pdf)/','',own).removesuffix('.pdf')
|
|
if re.fullmatch(r'(?:\d{4}\.\d{4,5}|[a-zA-Z.-]+/\d{7})(?:v\d+)?',own):identifiers['arxiv']=own
|
|
tags=[];index_fields={}
|
|
for field in ('keywords','primary_area','secondary_area','Please_choose_the_closest_area_that_your_submission_falls_into'):
|
|
raw=value(content,field)
|
|
if raw is None:continue
|
|
index_fields[field]=raw
|
|
terms=raw if isinstance(raw,list) else re.split(r'[,;\n]',raw) if field=='keywords' and isinstance(raw,str) else [raw]
|
|
namespace=SOURCE+':'+('keyword' if field=='keywords' else field)
|
|
tags.extend({'namespace':namespace,'value':term.strip()} for term in terms if isinstance(term,str) and term.strip())
|
|
ids=[{'scheme':'openreview','value':forum}]
|
|
if number:ids.append({'scheme':'iclr_number','value':str(number)})
|
|
return {'source':SOURCE,'external_id':f'{year}/{forum}','url':'https://openreview.net/forum?id='+forum,
|
|
'title':' '.join(title.split()),'abstract':abstract.strip() if abstract else None,'authors':authors,
|
|
'identifiers':identifiers,'submission_identifiers':ids,'venue':'ICLR','year':year,'track':'main',
|
|
'submitted_at':date_ms(note.get('cdate') or note.get('tcdate')),'observed_at':AS_OF,'tags':tags,
|
|
'metadata':{'forum':forum,'source_submission_number':number,'original_invitations':invitations(note),
|
|
'public_readers_verified':True,'index_fields':index_fields,'venue':value(content,'venue'),
|
|
'venueid':value(content,'venueid'),'snapshot_release':RELEASE,
|
|
'snapshot_observation_upper_bound':AS_OF}},None
|
|
|
|
|
|
def decision_record(note,year):
|
|
if not public(note):return None,'not_public'
|
|
base=f'ICLR.cc/{year}/Conference/'
|
|
if not any(i.startswith(base) and i.endswith('/Decision') for i in invitations(note)):return None,'not_decision'
|
|
if not any(isinstance(s,str) and s.startswith(base) and s.rsplit('/',1)[-1] in ('Program_Chairs','Area_Chairs','Senior_Area_Chairs') for s in note.get('signatures',[])):
|
|
return None,'unverified_signature'
|
|
if note.get('ddate'):return None,'deleted_decision'
|
|
forum=note.get('forum');identifier=note.get('id')
|
|
if not all(isinstance(v,str) and re.fullmatch(r'[A-Za-z0-9_-]+',v) for v in (forum,identifier)):return None,'invalid_identity'
|
|
if note.get('_paper_forum',forum)!=forum:return None,'forum_mismatch'
|
|
raw=value(note.get('content') or {},'decision')
|
|
if not isinstance(raw,str) or not raw.strip():return None,'missing_decision_value'
|
|
return {'id':identifier,'forum':forum,'status':decision_status(raw),'raw_value':raw,
|
|
'evidence_url':f'https://openreview.net/forum?id={forum}¬eId={identifier}',
|
|
'observed_at':AS_OF,'decision_date':date_ms(note.get('cdate') or note.get('tcdate') or note.get('mdate')),
|
|
'source_decision_id':identifier,'authority':{'invitations':invitations(note),
|
|
'signatures':note.get('signatures',[]),'public_readers_verified':True}},None
|
|
|
|
|
|
def collect(archive,out,years,expected_sha=SHA256):
|
|
actual=digest_file(Path(archive))
|
|
if actual['sha256']!=expected_sha:raise ValueError('Snapshot checksum mismatch before parsing')
|
|
if expected_sha==SHA256 and actual['bytes']!=512940722:raise ValueError('Unexpected pinned snapshot size')
|
|
out=Path(out);out.mkdir(parents=True,exist_ok=True);years=set(years)
|
|
counts=defaultdict(Counter);members=set()
|
|
with tempfile.TemporaryDirectory(prefix='openreview-stage-',dir=out) as tmp,closing(sqlite3.connect(Path(tmp)/'metadata.sqlite')) as c:
|
|
c.executescript('PRAGMA synchronous=OFF; CREATE TABLE papers(year INTEGER,forum TEXT,payload TEXT,PRIMARY KEY(year,forum)); CREATE TABLE decisions(year INTEGER,id TEXT,forum TEXT,payload TEXT,PRIMARY KEY(year,id)); CREATE INDEX decisions_forum ON decisions(year,forum);')
|
|
with tarfile.open(archive,mode='r|bz2') as tar:
|
|
for member in tar:
|
|
match=re.fullmatch(r'(?:\./)?data/iclr_(20\d{2})/(papers|reviews)\.jsonl',member.name)
|
|
if not match or int(match[1]) not in years:continue
|
|
year=int(match[1]);kind=match[2]
|
|
if not member.isfile() or (year,kind) in members:raise ValueError('Invalid or repeated metadata member')
|
|
members.add((year,kind));lines=0
|
|
with tar.extractfile(member) as stream:
|
|
while line:=stream.readline(16*1024**2+1):
|
|
if len(line)>16*1024**2:raise ValueError('Unexpectedly large metadata line')
|
|
if not line.strip():continue
|
|
lines+=1;counts[year][kind+'_lines']+=1
|
|
if kind=='reviews' and b'/Decision' not in line:continue
|
|
note=json.loads(line)
|
|
row,reason=paper_record(note,year) if kind=='papers' else decision_record(note,year)
|
|
if row is None:
|
|
counts[year][kind+'_excluded_'+reason]+=1;continue
|
|
if kind=='papers':
|
|
forum=row['metadata']['forum'];payload=json.dumps(row,ensure_ascii=False)
|
|
old=c.execute('SELECT payload FROM papers WHERE year=? AND forum=?',(year,forum)).fetchone()
|
|
if old and old[0]!=payload:raise ValueError('Conflicting copies of one submission')
|
|
if old:counts[year]['duplicate_papers']+=1
|
|
else:c.execute('INSERT INTO papers VALUES(?,?,?)',(year,forum,payload))
|
|
else:
|
|
payload=json.dumps(row,ensure_ascii=False);old=c.execute('SELECT payload FROM decisions WHERE year=? AND id=?',(year,row['id'])).fetchone()
|
|
if old and old[0]!=payload:raise ValueError('Conflicting copies of one decision note')
|
|
if old:counts[year]['duplicate_decisions']+=1
|
|
else:c.execute('INSERT INTO decisions VALUES(?,?,?,?)',(year,row['id'],row['forum'],payload))
|
|
if lines%5000==0:print(json.dumps({'source':SOURCE,'year':year,'member':kind,'lines':lines}),flush=True)
|
|
c.commit();print(json.dumps({'source':SOURCE,'year':year,'member':kind,'lines':lines,'status':'member_processed'}),flush=True)
|
|
if members!={(year,kind) for year in years for kind in ('papers','reviews')}:raise ValueError('Selected year metadata members are missing')
|
|
if expected_sha==SHA256:
|
|
for year in years:
|
|
if counts[year]['papers_lines']!=EXPECTED.get(year):raise ValueError('Pinned release paper count differs from published inventory')
|
|
status_path=out/'openreview-snapshot-year-status.json';catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
|
|
for year in sorted(years,reverse=True):
|
|
target=out/f'openreview-snapshot-{year}-main.jsonl';part=target.with_suffix('.part')
|
|
statuses=Counter();event_counts=Counter();missing=0;written=0
|
|
with part.open('w') as output:
|
|
for forum,payload in c.execute('SELECT forum,payload FROM papers WHERE year=? ORDER BY forum',(year,)):
|
|
row=json.loads(payload)
|
|
events=[json.loads(r[0]) for r in c.execute('SELECT payload FROM decisions WHERE year=? AND forum=?',(year,forum))]
|
|
events.sort(key=lambda event:(event['decision_date'] or '',event['id']))
|
|
row['decision_events']=events;row['status']=events[-1]['status'] if events else 'unknown'
|
|
row['metadata']['snapshot_sha256']=actual['sha256'];row['metadata']['decision_note_ids']=[e['id'] for e in events]
|
|
row['metadata']['decision_authority']=[{'note_id':e['id'],**e['authority']} for e in events]
|
|
output.write(json.dumps(row,ensure_ascii=False)+'\n');written+=1;missing+=not bool(row['abstract']);statuses[row['status']]+=1
|
|
event_counts.update(e['status'] for e in events)
|
|
if not written:raise ValueError('No verified public submissions for selected year')
|
|
unmatched=c.execute('SELECT count(*) FROM decisions d WHERE d.year=? AND NOT EXISTS(SELECT 1 FROM papers p WHERE p.year=d.year AND p.forum=d.forum)',(year,)).fetchone()[0]
|
|
unmatched_path=None
|
|
if unmatched:
|
|
unresolved=out/f'openreview-snapshot-{year}-unmatched-decisions.jsonl'
|
|
with unresolved.with_suffix('.part').open('w') as output:
|
|
for payload, in c.execute('SELECT payload FROM decisions d WHERE d.year=? AND NOT EXISTS(SELECT 1 FROM papers p WHERE p.year=d.year AND p.forum=d.forum)',(year,)):
|
|
event=json.loads(payload);event.update(year=year,reason='public decision lacks public submission metadata in snapshot',snapshot_sha256=actual['sha256'])
|
|
output.write(json.dumps(event,ensure_ascii=False)+'\n')
|
|
unresolved.with_suffix('.part').replace(unresolved);unmatched_path=str(unresolved)
|
|
part.replace(target)
|
|
manifest={'source':SOURCE,'parser_version':1,'venue':'ICLR','year':year,'track':'main','count':written,
|
|
'list_url':RELEASE,'snapshot_sha256':actual['sha256'],'snapshot_observation_upper_bound':AS_OF,
|
|
'missing_abstract_count':missing,'submission_status_counts':dict(statuses),'decision_event_counts':dict(event_counts),
|
|
'unmatched_public_decisions':unmatched,'archive_counts':dict(counts[year]),'file':str(target),
|
|
'unmatched_evidence_file':unmatched_path,
|
|
'status':'partial' if missing or statuses['unknown'] or unmatched or any('_excluded_' in k for k in counts[year]) else 'collected'}
|
|
target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));catalogue[str(year)]=manifest
|
|
print(json.dumps(manifest),flush=True)
|
|
stage=status_path.with_suffix('.part');stage.write_text(json.dumps(catalogue,indent=2));stage.replace(status_path)
|
|
(out/'openreview-snapshot-last-run.json').write_text(json.dumps({'requested_years':sorted(years),'sha256':actual['sha256'],'release':RELEASE,'observed_at_upper_bound':AS_OF},indent=2))
|
|
return {year:catalogue[str(year)] for year in years}
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--archive',required=True);parser.add_argument('--out',required=True)
|
|
parser.add_argument('--year',type=int,action='append');args=parser.parse_args(argv)
|
|
collect(args.archive,args.out,args.year or sorted(EXPECTED))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|