"""Official NeurIPS program metadata, with presentations separate from submissions.""" import argparse from collections import Counter, defaultdict import fcntl import hashlib import json from pathlib import Path import re import tempfile from urllib.parse import parse_qs, urlsplit from paper_atlas.db.submissions import timestamp from paper_atlas.io.http import fetch URL = 'https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json' GROUPS = { 'https://openreview.net/group?id=NeurIPS.cc/2026/Conference': 'main', 'https://openreview.net/group?id=NeurIPS.cc/2026/Evaluations_and_Datasets_Track': 'evaluations_datasets', 'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track': 'position', } ACCEPTED = {'Accept', 'Accept (poster)', 'Accept (spotlight)', 'Accept (oral)', 'Accept (Spotlight)', 'Accept (Oral)'} LIMIT = 64*1024**2 def forum_id(url): if not isinstance(url, str): return None try: parsed = urlsplit(url) except ValueError: return None query = parse_qs(parsed.query, keep_blank_values=True) if (parsed.scheme != 'https' or parsed.netloc != 'openreview.net' or parsed.path != '/forum' or parsed.fragment or set(query) != {'id'} or len(query['id']) != 1 or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', query['id'][0])): return None return query['id'][0] def parse_program(data, evidence): root = json.loads(data) if (not isinstance(root, dict) or not {'count', 'next', 'previous', 'results'} <= set(root) or not isinstance(root.get('results'), list) or type(root.get('count')) is not int or root['count'] != len(root['results']) or root.get('next') is not None or root.get('previous') is not None): raise ValueError('Expected one complete, nonpaginated public program snapshot') if not root['results']: raise ValueError('Empty program snapshot') events, grouped, held, duplicates = {}, defaultdict(list), [], [] for index, event in enumerate(root['results']): if not isinstance(event, dict) or type(event.get('id')) is not int or event['id'] <= 0: raise ValueError('Missing stable event ID') eid = event['id'] if eid in events: first_index, first = events[eid] if first != event: raise ValueError('Conflicting payloads share one publisher event ID') duplicates.append({'event_id': eid, 'first_row': first_index, 'duplicate_row': index}) continue events[eid] = index, event origin = event.get('sourceurl') reason = None if not isinstance(origin, str) or origin not in GROUPS: reason = ('journal_presentation_requires_original_publication_relation' if isinstance(origin, str) and origin.startswith('journal-track/') else 'bridging_future_scope_requires_review' if origin == 'neurips2026-bridging-the-future' else 'unreviewed_or_missing_submission_origin') elif event.get('visible') is not True: reason = 'program_visibility_requires_review' elif not isinstance(event.get('decision'), str) or event['decision'] not in ACCEPTED: reason = 'unreviewed_decision_label' elif event.get('event_type') not in ('Poster', 'Oral'): reason = 'unreviewed_presentation_type' forum = forum_id(event.get('paper_url')) if reason is None and forum is None: reason = 'missing_or_invalid_original_submission_link' if reason: held.append({'row': index, 'event_id': eid, 'reason': reason, 'raw_event': event}) continue if (not isinstance(event.get('name'), str) or not event['name'].strip() or not isinstance(event.get('authors'), list) or not event['authors'] or any(not isinstance(a, dict) or not isinstance(a.get('fullname'), str) or not a['fullname'].strip() for a in event['authors'])): raise ValueError('Empty title or incomplete ordered author list') if not isinstance(event.get('virtualsite_url'), str) or not re.fullmatch( r'/virtual/2026/(?:poster|oral)/'+str(eid), event['virtualsite_url']): raise ValueError('Publisher event URL differs from year/type/ID') grouped[forum].append((index, event)) records = [] folded = 0 for forum, observations in sorted(grouped.items()): # Signed sourceid values are preserved verbatim in the presentation # evidence; they are not independent submissions or scientific works. signatures = {json.dumps([e['name'], e['authors'], e['sourceurl'], e['decision']], ensure_ascii=False, sort_keys=True) for _, e in observations} if len(signatures) != 1: raise ValueError('Title, authors, origin or decision disagree for one forum') posters = [e for _, e in observations if e['event_type'] == 'Poster'] event = posters[0] if posters else observations[0][1] track = GROUPS[event['sourceurl']] presentations = [] for index, e in observations: presentations.append({'row': index, 'event_id': e['id'], 'uid': e.get('uid'), 'sourceid': e.get('sourceid'), 'event_type': e['event_type'], 'eventtype': e.get('eventtype'), 'program_url': e['virtualsite_url'], 'session': e.get('session'), 'starttime': e.get('starttime'), 'endtime': e.get('endtime'), 'parent_id': e.get('parent_id'), 'raw_decision': e['decision']}) formats = sorted({e['event_type'] for _, e in observations}) records.append({ 'source': 'neurips_program', 'external_id': '2026/'+forum, 'url': event['paper_url'], 'title': event['name'], 'abstract': None, 'authors': [a['fullname'] for a in event['authors']], 'identifiers': {'openreview': forum}, 'venue': 'NeurIPS', 'venue_aliases': [], 'year': 2026, 'track': track, 'status': 'accepted', 'observed_at': evidence['observed_at'], 'raw_decision': event['decision'], 'metadata': {'source_group_url': event['sourceurl'], 'forum_id': forum, 'author_affiliations': [a.get('institution') for a in event['authors']], 'presentations': presentations, 'source_evidence': evidence, 'publication_evidence': 'Explicit accepted label in official conference program; not an original decision note or proof of proceedings publication.', 'decision_date_precision': 'not_supplied', 'missing_abstract_reason': 'Program JSON does not contain abstracts; paper HTML/PDF not fetched by this command.', 'identity_scope_note': 'One record per original forum, not one scientific work per event. Other venues/versions require independent identity review.'}, 'tags': [{'namespace': 'neurips_program:presentation_format', 'value': value} for value in formats]}) folded += len(observations)-1 if not records: raise ValueError('No reviewed submission origins found') if root['count'] != len(records)+len(held)+len(duplicates)+folded: raise ValueError('Program partition failed') return records, held, duplicates, folded def read_program(cache, snapshots): if bool(cache) == bool(snapshots): raise ValueError('Choose exactly one cache or frozen snapshot directory') if snapshots: base = Path(snapshots).resolve() matches = [r for r in json.loads((base/'receipts.json').read_text())['receipts'] if r['url'] == URL] if len(matches) != 1: raise ValueError('Expected one official program receipt') r = matches[0]; filename = Path(r['file']) if (filename.is_absolute() or '..' in filename.parts or not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200): raise ValueError('Unsafe or unsuccessful frozen program response') data = (base/filename).read_bytes() if len(data) > LIMIT or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']: raise ValueError('Frozen program checksum/size mismatch') observed = r['finished_at'] else: data = fetch(URL, cache, LIMIT, delay=1) r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text()) observed = r['retrieved_at'] timestamp(observed) return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data), 'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.json', 'observed_at': observed, 'http_status': 200, 'response_receipt': r} def collect(out, *, cache=None, snapshots=None): out = Path(out); out.mkdir(parents=True, exist_ok=True) report_name = 'neurips-program-2026.json' with (out/'neurips-program.collect.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (out/report_name).exists(): raise ValueError('Completed collection exists; use a new directory and reconcile before refresh') data, evidence = read_program(cache, snapshots) records, held, duplicates, folded = parse_program(data, evidence) groups = defaultdict(list) for r in records: groups[r['track']].append(r) files = [] with tempfile.TemporaryDirectory(prefix='.neurips-program-', dir=out) as temporary: stage = Path(temporary); (stage/'evidence').mkdir() (stage/evidence['file']).write_bytes(data) for track, group in sorted(groups.items()): p = stage/f'neurips-program-2026-{track}.jsonl' p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group)) manifest = {'source': 'neurips_program', 'parser_version': 1, 'venue': 'NeurIPS', 'year': 2026, 'track': track, 'count': len(group), 'file': str(out/p.name), 'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL, 'status': 'partial', 'missing_abstract_count': len(group), 'note': 'Reviewed forum-level program partition; not full current proceedings, abstracts, public rejection coverage or scientific-work dedup.'} p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n') files.append(manifest) result = {'source': 'neurips_program', 'year': 2026, 'finished_at': timestamp(), 'status': 'collected_pending_identity_review', 'records': len(records), 'files': files, 'held_presentations': held, 'exact_duplicate_event_rows': duplicates, 'folded_extra_presentation_rows': folded, 'source_receipts': [evidence], 'database_written': False, 'pdfs_downloaded': False, 'note': 'Public program only; rejected opt-in availability needs a separate source. Journals, bridging and missing origins remain held; absence is not rejection.'} (stage/report_name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') (out/'evidence').mkdir(exist_ok=True) for p in (stage/'evidence').iterdir(): p.replace(out/'evidence'/p.name) for p in stage.iterdir(): if p.is_file() and p.name != report_name: p.replace(out/p.name) (stage/report_name).replace(out/report_name) return result def main(argv=None, *, prog=None): parser = argparse.ArgumentParser(prog=prog) parser.add_argument('--out', required=True) source = parser.add_mutually_exclusive_group(required=True) source.add_argument('--cache') source.add_argument('--snapshots') args = parser.parse_args(argv) result = collect(args.out, cache=args.cache, snapshots=args.snapshots) print(json.dumps({'source': result['source'], 'year': 2026, 'status': result['status'], 'records': result['records'], 'files': result['files'], 'held_presentations': len(result['held_presentations']), 'exact_duplicate_event_rows': len(result['exact_duplicate_event_rows']), 'folded_extra_presentation_rows': result['folded_extra_presentation_rows'], 'report_file': str(Path(args.out)/'neurips-program-2026.json'), 'database_written': False, 'pdfs_downloaded': False}, ensure_ascii=False))