219 lines
13 KiB
Python
219 lines
13 KiB
Python
"""Official NeurIPS program metadata, with presentations separate from submissions."""
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
from paper_atlas.db.submissions import timestamp
|
|
from paper_atlas.io.http import fetch
|
|
|
|
URL = 'https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json'
|
|
GROUPS = {
|
|
'https://openreview.net/group?id=NeurIPS.cc/2026/Conference': 'main',
|
|
'https://openreview.net/group?id=NeurIPS.cc/2026/Evaluations_and_Datasets_Track': 'evaluations_datasets',
|
|
'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track': 'position',
|
|
}
|
|
ACCEPTED = {'Accept', 'Accept (poster)', 'Accept (spotlight)', 'Accept (oral)',
|
|
'Accept (Spotlight)', 'Accept (Oral)'}
|
|
LIMIT = 64*1024**2
|
|
|
|
|
|
def forum_id(url):
|
|
if not isinstance(url, str):
|
|
return None
|
|
try:
|
|
parsed = urlsplit(url)
|
|
except ValueError:
|
|
return None
|
|
query = parse_qs(parsed.query, keep_blank_values=True)
|
|
if (parsed.scheme != 'https' or parsed.netloc != 'openreview.net' or
|
|
parsed.path != '/forum' or parsed.fragment or set(query) != {'id'} or
|
|
len(query['id']) != 1 or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', query['id'][0])):
|
|
return None
|
|
return query['id'][0]
|
|
|
|
|
|
def parse_program(data, evidence):
|
|
root = json.loads(data)
|
|
if (not isinstance(root, dict) or not {'count', 'next', 'previous', 'results'} <= set(root) or
|
|
not isinstance(root.get('results'), list) or
|
|
type(root.get('count')) is not int or root['count'] != len(root['results']) or
|
|
root.get('next') is not None or root.get('previous') is not None):
|
|
raise ValueError('Expected one complete, nonpaginated public program snapshot')
|
|
if not root['results']:
|
|
raise ValueError('Empty program snapshot')
|
|
events, grouped, held, duplicates = {}, defaultdict(list), [], []
|
|
for index, event in enumerate(root['results']):
|
|
if not isinstance(event, dict) or type(event.get('id')) is not int or event['id'] <= 0:
|
|
raise ValueError('Missing stable event ID')
|
|
eid = event['id']
|
|
if eid in events:
|
|
first_index, first = events[eid]
|
|
if first != event:
|
|
raise ValueError('Conflicting payloads share one publisher event ID')
|
|
duplicates.append({'event_id': eid, 'first_row': first_index, 'duplicate_row': index})
|
|
continue
|
|
events[eid] = index, event
|
|
origin = event.get('sourceurl')
|
|
reason = None
|
|
if not isinstance(origin, str) or origin not in GROUPS:
|
|
reason = ('journal_presentation_requires_original_publication_relation'
|
|
if isinstance(origin, str) and origin.startswith('journal-track/') else
|
|
'bridging_future_scope_requires_review' if origin == 'neurips2026-bridging-the-future' else
|
|
'unreviewed_or_missing_submission_origin')
|
|
elif event.get('visible') is not True:
|
|
reason = 'program_visibility_requires_review'
|
|
elif not isinstance(event.get('decision'), str) or event['decision'] not in ACCEPTED:
|
|
reason = 'unreviewed_decision_label'
|
|
elif event.get('event_type') not in ('Poster', 'Oral'):
|
|
reason = 'unreviewed_presentation_type'
|
|
forum = forum_id(event.get('paper_url'))
|
|
if reason is None and forum is None:
|
|
reason = 'missing_or_invalid_original_submission_link'
|
|
if reason:
|
|
held.append({'row': index, 'event_id': eid, 'reason': reason, 'raw_event': event})
|
|
continue
|
|
if (not isinstance(event.get('name'), str) or not event['name'].strip() or
|
|
not isinstance(event.get('authors'), list) or not event['authors'] or
|
|
any(not isinstance(a, dict) or not isinstance(a.get('fullname'), str) or
|
|
not a['fullname'].strip() for a in event['authors'])):
|
|
raise ValueError('Empty title or incomplete ordered author list')
|
|
if not isinstance(event.get('virtualsite_url'), str) or not re.fullmatch(
|
|
r'/virtual/2026/(?:poster|oral)/'+str(eid), event['virtualsite_url']):
|
|
raise ValueError('Publisher event URL differs from year/type/ID')
|
|
grouped[forum].append((index, event))
|
|
records = []
|
|
folded = 0
|
|
for forum, observations in sorted(grouped.items()):
|
|
# Signed sourceid values are preserved verbatim in the presentation
|
|
# evidence; they are not independent submissions or scientific works.
|
|
signatures = {json.dumps([e['name'], e['authors'], e['sourceurl'], e['decision']],
|
|
ensure_ascii=False, sort_keys=True) for _, e in observations}
|
|
if len(signatures) != 1:
|
|
raise ValueError('Title, authors, origin or decision disagree for one forum')
|
|
posters = [e for _, e in observations if e['event_type'] == 'Poster']
|
|
event = posters[0] if posters else observations[0][1]
|
|
track = GROUPS[event['sourceurl']]
|
|
presentations = []
|
|
for index, e in observations:
|
|
presentations.append({'row': index, 'event_id': e['id'], 'uid': e.get('uid'),
|
|
'sourceid': e.get('sourceid'), 'event_type': e['event_type'],
|
|
'eventtype': e.get('eventtype'), 'program_url': e['virtualsite_url'],
|
|
'session': e.get('session'), 'starttime': e.get('starttime'),
|
|
'endtime': e.get('endtime'), 'parent_id': e.get('parent_id'),
|
|
'raw_decision': e['decision']})
|
|
formats = sorted({e['event_type'] for _, e in observations})
|
|
records.append({
|
|
'source': 'neurips_program', 'external_id': '2026/'+forum,
|
|
'url': event['paper_url'], 'title': event['name'], 'abstract': None,
|
|
'authors': [a['fullname'] for a in event['authors']],
|
|
'identifiers': {'openreview': forum}, 'venue': 'NeurIPS', 'venue_aliases': [],
|
|
'year': 2026, 'track': track, 'status': 'accepted',
|
|
'observed_at': evidence['observed_at'], 'raw_decision': event['decision'],
|
|
'metadata': {'source_group_url': event['sourceurl'], 'forum_id': forum,
|
|
'author_affiliations': [a.get('institution') for a in event['authors']],
|
|
'presentations': presentations, 'source_evidence': evidence,
|
|
'publication_evidence': 'Explicit accepted label in official conference program; not an original decision note or proof of proceedings publication.',
|
|
'decision_date_precision': 'not_supplied',
|
|
'missing_abstract_reason': 'Program JSON does not contain abstracts; paper HTML/PDF not fetched by this command.',
|
|
'identity_scope_note': 'One record per original forum, not one scientific work per event. Other venues/versions require independent identity review.'},
|
|
'tags': [{'namespace': 'neurips_program:presentation_format', 'value': value}
|
|
for value in formats]})
|
|
folded += len(observations)-1
|
|
if not records:
|
|
raise ValueError('No reviewed submission origins found')
|
|
if root['count'] != len(records)+len(held)+len(duplicates)+folded:
|
|
raise ValueError('Program partition failed')
|
|
return records, held, duplicates, folded
|
|
|
|
|
|
def read_program(cache, snapshots):
|
|
if bool(cache) == bool(snapshots):
|
|
raise ValueError('Choose exactly one cache or frozen snapshot directory')
|
|
if snapshots:
|
|
base = Path(snapshots).resolve()
|
|
matches = [r for r in json.loads((base/'receipts.json').read_text())['receipts'] if r['url'] == URL]
|
|
if len(matches) != 1:
|
|
raise ValueError('Expected one official program receipt')
|
|
r = matches[0]; filename = Path(r['file'])
|
|
if (filename.is_absolute() or '..' in filename.parts or
|
|
not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200):
|
|
raise ValueError('Unsafe or unsuccessful frozen program response')
|
|
data = (base/filename).read_bytes()
|
|
if len(data) > LIMIT or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
|
|
raise ValueError('Frozen program checksum/size mismatch')
|
|
observed = r['finished_at']
|
|
else:
|
|
data = fetch(URL, cache, LIMIT, delay=1)
|
|
r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text())
|
|
observed = r['retrieved_at']
|
|
timestamp(observed)
|
|
return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
|
|
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.json',
|
|
'observed_at': observed, 'http_status': 200, 'response_receipt': r}
|
|
|
|
|
|
def collect(out, *, cache=None, snapshots=None):
|
|
out = Path(out); out.mkdir(parents=True, exist_ok=True)
|
|
report_name = 'neurips-program-2026.json'
|
|
with (out/'neurips-program.collect.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (out/report_name).exists():
|
|
raise ValueError('Completed collection exists; use a new directory and reconcile before refresh')
|
|
data, evidence = read_program(cache, snapshots)
|
|
records, held, duplicates, folded = parse_program(data, evidence)
|
|
groups = defaultdict(list)
|
|
for r in records:
|
|
groups[r['track']].append(r)
|
|
files = []
|
|
with tempfile.TemporaryDirectory(prefix='.neurips-program-', dir=out) as temporary:
|
|
stage = Path(temporary); (stage/'evidence').mkdir()
|
|
(stage/evidence['file']).write_bytes(data)
|
|
for track, group in sorted(groups.items()):
|
|
p = stage/f'neurips-program-2026-{track}.jsonl'
|
|
p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
|
|
manifest = {'source': 'neurips_program', 'parser_version': 1, 'venue': 'NeurIPS',
|
|
'year': 2026, 'track': track, 'count': len(group), 'file': str(out/p.name),
|
|
'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL,
|
|
'status': 'partial', 'missing_abstract_count': len(group),
|
|
'note': 'Reviewed forum-level program partition; not full current proceedings, abstracts, public rejection coverage or scientific-work dedup.'}
|
|
p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
|
|
files.append(manifest)
|
|
result = {'source': 'neurips_program', 'year': 2026, 'finished_at': timestamp(),
|
|
'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
|
|
'held_presentations': held, 'exact_duplicate_event_rows': duplicates,
|
|
'folded_extra_presentation_rows': folded, 'source_receipts': [evidence],
|
|
'database_written': False, 'pdfs_downloaded': False,
|
|
'note': 'Public program only; rejected opt-in availability needs a separate source. Journals, bridging and missing origins remain held; absence is not rejection.'}
|
|
(stage/report_name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
|
(out/'evidence').mkdir(exist_ok=True)
|
|
for p in (stage/'evidence').iterdir():
|
|
p.replace(out/'evidence'/p.name)
|
|
for p in stage.iterdir():
|
|
if p.is_file() and p.name != report_name:
|
|
p.replace(out/p.name)
|
|
(stage/report_name).replace(out/report_name)
|
|
return result
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--out', required=True)
|
|
source = parser.add_mutually_exclusive_group(required=True)
|
|
source.add_argument('--cache')
|
|
source.add_argument('--snapshots')
|
|
args = parser.parse_args(argv)
|
|
result = collect(args.out, cache=args.cache, snapshots=args.snapshots)
|
|
print(json.dumps({'source': result['source'], 'year': 2026, 'status': result['status'],
|
|
'records': result['records'], 'files': result['files'],
|
|
'held_presentations': len(result['held_presentations']),
|
|
'exact_duplicate_event_rows': len(result['exact_duplicate_event_rows']),
|
|
'folded_extra_presentation_rows': result['folded_extra_presentation_rows'],
|
|
'report_file': str(Path(args.out)/'neurips-program-2026.json'),
|
|
'database_written': False, 'pdfs_downloaded': False}, ensure_ascii=False))
|