diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 06f38e3..bc26949 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -50,6 +50,16 @@ jobs: subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True) + neurips_snapshots = root/'neurips2026-program-snapshots' + neurips_snapshots.mkdir() + (neurips_snapshots/'program-json.response').write_bytes(gzip.decompress((src/'tests/fixtures/neurips2026-program.json.gz').read_bytes())) + (neurips_snapshots/'receipts.json').write_bytes((src/'tests/fixtures/neurips2026-receipts.json').read_bytes()) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--snapshots',str(neurips_snapshots),'--out',str(root/'installed-neurips2026-program')],cwd=outside,check=True) + neurips_report = json.loads((root/'installed-neurips2026-program/neurips-program-2026.json').read_text()) + assert neurips_report['records'] == 9004 and len(neurips_report['held_presentations']) == 139 + assert len(neurips_report['exact_duplicate_event_rows']) == 1 and neurips_report['folded_extra_presentation_rows'] == 132 + assert not neurips_report['database_written'] and not neurips_report['pdfs_downloaded'] subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index 4fe12d1..50a3908 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \ --out /data/paper-index/collected ``` +NeurIPS 2026 的[官方节目采集器](docs/neurips-program.md)按 main、evaluations/datasets、position 分开保存，合并同一 forum 的口头与海报展示，保留期刊等未决关系；运行 `paper-atlas collect neurips-program`。它只生成元数据文件，身份核验和生产导入另行执行。 + 早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别，保留原名与范围证据；代码支持不代表已补齐生产数据。 EWSL、EPS、ECP、EWSP 的[具体前身卷与完整快照要求](docs/coverage/historical-predecessors-2026-11-09.md)单独核验，保留原始系列及事件/出版年份；旧候选文件缺少 ECP/EWSP 时会明确提示，避免静默遗漏。 diff --git a/docs/neurips-program.md b/docs/neurips-program.md new file mode 100644 index 0000000..df25f52 --- /dev/null +++ b/docs/neurips-program.md @@ -0,0 +1,56 @@ +# NeurIPS 官方节目元数据 + +`paper-atlas collect neurips-program` 是按需运行的采集器，当前核验范围是 2026 年的[官方 orals/posters JSON](https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json)。它只请求这个固定公开元数据文件，不连接数据库，不请求 OpenReview API，不下载 PDF、视频或个人资料页。模块位于 `paper_atlas.collectors.neurips_program`，通过统一 CLI 使用。 + +## 已核验快照 + +2026-11-09T17:57:40Z 冻结的原始响应共 20,447,021 字节，SHA256 `879293a863057fa05370443b3b5ddee12a10eeb37c810abc6c1d53f71f545997`。原始 `count=9276` 与结果数组长度一致，`next`、`previous` 均为空；这只证明此份公开文件已完整取得，不是全部会议出版物或公开拒稿全集。 + +按官方逐条 `sourceurl` 中的明确投稿来源分开处理： + +| 原始 OpenReview group | 数据库 track | 不同 forum 数 | +| --- | --- | ---: | +| `NeurIPS.cc/2026/Conference` | `main` | 7896 | +| `NeurIPS.cc/2026/Evaluations_and_Datasets_Track` | `evaluations_datasets` | 971 | +| `NeurIPS.cc/2026/Position_Paper_Track` | `position` | 137 | +| 合计 | | 9004 | + +只接受已核验的显式 `Accept`、`Accept (poster/spotlight/oral)` 标签，保留原始大小写和文本。未知来源、决定、展示类型、可见性或无法验证的 forum 链接进入 `held_presentations`，不推成 accepted 或 rejected。空标题、缺少作者、相同 event ID 不同内容、同一 forum 的题名/作者/来源/决定冲突会停止采集。 + +原 9276 行中，event 154866 有一条内容完全相同的重复行，单独保留其原始行号；另外 132 个口头展示与同 forum 的海报合并为一条投稿候选，完整保留两种展示及时间、场次、原始带符号 `sourceid`。负数 `sourceid` 不能被解释为拒稿或另一次投稿，也不转换为全局论文标识。 + +其余 139 行单独留档：87 个期刊展示、36 个 Bridging the Future 展示、16 个未提供明确投稿来源的条目。这些不因缺少链接而被推为拒稿，也不能被冒充为独立主会审稿录用。后续须核实原期刊发表、认证、复现或其他展示关系。 + +9004 条是不同 forum 的来源记录，不是已经新增 9004 篇独立科学论文。不同 forum 即使题名和作者相同也保留各自投稿；跨会及版本关系另行核验。题名、完整作者顺序和机构取自原始 JSON，口头与海报不是研究类型；展示标签独立使用 `neurips_program:presentation_format` namespace。节目中的未来展示时间不冒充决定时间。 + +## 运行与恢复 + +在线模式只请求一个固定元数据 URL，并复用受限大小的响应缓存： + +```sh +paper-atlas collect neurips-program \ + --cache /data/paper-index/cache/neurips2026-program \ + --out /data/paper-index/collected/neurips2026-program-YYYYMMDD +``` + +冻结模式只读 `receipts.json` 对应的原始文件： + +```sh +paper-atlas collect neurips-program \ + --snapshots /data/paper-index/collected/neurips2026-program-probe-v1 \ + --out /data/paper-index/collected/neurips2026-program-normalized-v1 +``` + +快照回执包含精确 `url`、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区的 `finished_at`。拒绝越界路径/符号链接、重复回执、失败响应、大小或哈希不符。在线模式沿用项目 HTTP 缓存和限速，验证回执时间后保存原始 JSON。输出目录有互斥锁，完成报告最后发布；已有完成报告时要求新建输出目录，避免覆盖后误重放。 + +输出三个 track 的 JSONL/manifest、原始 JSON 证据和 `neurips-program-2026.json`。manifest 始终为 `partial`；原始节目 JSON 没有摘要，9004 条 `abstract=null` 和缺失原因如实保存，不能用别篇摘要填充。命令终端只输出数量与路径，完整待处理展示与原始行号在报告中。 + +恢复时先校验文件和回执，使用同一完整快照生成新输出；若已有完成报告或导入回执，先核实实际来源绑定和 checkpoint，不重放已消费队列。生成文件后还需 PostgreSQL 身份预检、关联审核、增量队列、原生导入和独立验收；源文件成功、CI 通过均不能替代生产导入或最终全库去重。 + +## 目录差异与公开拒稿限制 + +同期[官网 Downloads 页面](https://neurips.cc/Downloads/2026)的活动总数不是本 JSON 的论文总数。逐个 poster ID/题名对照后，网页另有 event 155165（Lemon）未出现在本 JSON 中，须单独取得并审核元数据，不能补造其 track 或 forum。16 个未提供来源的条目在 JSON 中使用通用 session URL，而网页有各自 poster URL；原值保留，不能因 URL 映射差异制造新论文。 + +[2026 Main Track Handbook](https://neurips.cc/Conferences/2026/MainTrackHandbook)明确允许 rejected 作者选择公开。当前无账号公开 notes API 请求返回 `ChallengeRequiredError`、HTTP 403，失败原文和回执保留，不绕过验证；因此不能把本 accepted 节目文件当作拒稿数据源或据此推断拒稿是否存在。官网下载表单的 JSON 导出也曾超时，未取得导出内容，已改用独立公开静态节目文件并保留失败证据，不重试超时导出。 + +当前主动补漏范围从 2020 年起。期刊/展示关系、网页独有条目、摘要和公开负面决定仍有待办；冻结资料和代码适配不代表完整会议覆盖。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 081ec11..f2601d9 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -26,6 +26,7 @@ COMMANDS = { ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'), + ('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'), ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'), ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'), ('collect','dblp-family'): ('paper_atlas.collectors.dblp_recovery', {}, '从固定快照恢复指定 DBLP family 的原始候选'), diff --git a/paper_atlas/collectors/neurips_program.py b/paper_atlas/collectors/neurips_program.py new file mode 100644 index 0000000..1d87e7d --- /dev/null +++ b/paper_atlas/collectors/neurips_program.py @@ -0,0 +1,218 @@ +"""Official NeurIPS program metadata, with presentations separate from submissions.""" +import argparse +from collections import Counter, defaultdict +import fcntl +import hashlib +import json +from pathlib import Path +import re +import tempfile +from urllib.parse import parse_qs, urlsplit + +from paper_atlas.db.submissions import timestamp +from paper_atlas.io.http import fetch + +URL = 'https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json' +GROUPS = { + 'https://openreview.net/group?id=NeurIPS.cc/2026/Conference': 'main', + 'https://openreview.net/group?id=NeurIPS.cc/2026/Evaluations_and_Datasets_Track': 'evaluations_datasets', + 'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track': 'position', +} +ACCEPTED = {'Accept', 'Accept (poster)', 'Accept (spotlight)', 'Accept (oral)', + 'Accept (Spotlight)', 'Accept (Oral)'} +LIMIT = 64*1024**2 + + +def forum_id(url): + if not isinstance(url, str): + return None + try: + parsed = urlsplit(url) + except ValueError: + return None + query = parse_qs(parsed.query, keep_blank_values=True) + if (parsed.scheme != 'https' or parsed.netloc != 'openreview.net' or + parsed.path != '/forum' or parsed.fragment or set(query) != {'id'} or + len(query['id']) != 1 or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', query['id'][0])): + return None + return query['id'][0] + + +def parse_program(data, evidence): + root = json.loads(data) + if (not isinstance(root, dict) or not {'count', 'next', 'previous', 'results'} <= set(root) or + not isinstance(root.get('results'), list) or + type(root.get('count')) is not int or root['count'] != len(root['results']) or + root.get('next') is not None or root.get('previous') is not None): + raise ValueError('Expected one complete, nonpaginated public program snapshot') + if not root['results']: + raise ValueError('Empty program snapshot') + events, grouped, held, duplicates = {}, defaultdict(list), [], [] + for index, event in enumerate(root['results']): + if not isinstance(event, dict) or type(event.get('id')) is not int or event['id'] <= 0: + raise ValueError('Missing stable event ID') + eid = event['id'] + if eid in events: + first_index, first = events[eid] + if first != event: + raise ValueError('Conflicting payloads share one publisher event ID') + duplicates.append({'event_id': eid, 'first_row': first_index, 'duplicate_row': index}) + continue + events[eid] = index, event + origin = event.get('sourceurl') + reason = None + if not isinstance(origin, str) or origin not in GROUPS: + reason = ('journal_presentation_requires_original_publication_relation' + if isinstance(origin, str) and origin.startswith('journal-track/') else + 'bridging_future_scope_requires_review' if origin == 'neurips2026-bridging-the-future' else + 'unreviewed_or_missing_submission_origin') + elif event.get('visible') is not True: + reason = 'program_visibility_requires_review' + elif not isinstance(event.get('decision'), str) or event['decision'] not in ACCEPTED: + reason = 'unreviewed_decision_label' + elif event.get('event_type') not in ('Poster', 'Oral'): + reason = 'unreviewed_presentation_type' + forum = forum_id(event.get('paper_url')) + if reason is None and forum is None: + reason = 'missing_or_invalid_original_submission_link' + if reason: + held.append({'row': index, 'event_id': eid, 'reason': reason, 'raw_event': event}) + continue + if (not isinstance(event.get('name'), str) or not event['name'].strip() or + not isinstance(event.get('authors'), list) or not event['authors'] or + any(not isinstance(a, dict) or not isinstance(a.get('fullname'), str) or + not a['fullname'].strip() for a in event['authors'])): + raise ValueError('Empty title or incomplete ordered author list') + if not isinstance(event.get('virtualsite_url'), str) or not re.fullmatch( + r'/virtual/2026/(?:poster|oral)/'+str(eid), event['virtualsite_url']): + raise ValueError('Publisher event URL differs from year/type/ID') + grouped[forum].append((index, event)) + records = [] + folded = 0 + for forum, observations in sorted(grouped.items()): + # Signed sourceid values are preserved verbatim in the presentation + # evidence; they are not independent submissions or scientific works. + signatures = {json.dumps([e['name'], e['authors'], e['sourceurl'], e['decision']], + ensure_ascii=False, sort_keys=True) for _, e in observations} + if len(signatures) != 1: + raise ValueError('Title, authors, origin or decision disagree for one forum') + posters = [e for _, e in observations if e['event_type'] == 'Poster'] + event = posters[0] if posters else observations[0][1] + track = GROUPS[event['sourceurl']] + presentations = [] + for index, e in observations: + presentations.append({'row': index, 'event_id': e['id'], 'uid': e.get('uid'), + 'sourceid': e.get('sourceid'), 'event_type': e['event_type'], + 'eventtype': e.get('eventtype'), 'program_url': e['virtualsite_url'], + 'session': e.get('session'), 'starttime': e.get('starttime'), + 'endtime': e.get('endtime'), 'parent_id': e.get('parent_id'), + 'raw_decision': e['decision']}) + formats = sorted({e['event_type'] for _, e in observations}) + records.append({ + 'source': 'neurips_program', 'external_id': '2026/'+forum, + 'url': event['paper_url'], 'title': event['name'], 'abstract': None, + 'authors': [a['fullname'] for a in event['authors']], + 'identifiers': {'openreview': forum}, 'venue': 'NeurIPS', 'venue_aliases': [], + 'year': 2026, 'track': track, 'status': 'accepted', + 'observed_at': evidence['observed_at'], 'raw_decision': event['decision'], + 'metadata': {'source_group_url': event['sourceurl'], 'forum_id': forum, + 'author_affiliations': [a.get('institution') for a in event['authors']], + 'presentations': presentations, 'source_evidence': evidence, + 'publication_evidence': 'Explicit accepted label in official conference program; not an original decision note or proof of proceedings publication.', + 'decision_date_precision': 'not_supplied', + 'missing_abstract_reason': 'Program JSON does not contain abstracts; paper HTML/PDF not fetched by this command.', + 'identity_scope_note': 'One record per original forum, not one scientific work per event. Other venues/versions require independent identity review.'}, + 'tags': [{'namespace': 'neurips_program:presentation_format', 'value': value} + for value in formats]}) + folded += len(observations)-1 + if not records: + raise ValueError('No reviewed submission origins found') + if root['count'] != len(records)+len(held)+len(duplicates)+folded: + raise ValueError('Program partition failed') + return records, held, duplicates, folded + + +def read_program(cache, snapshots): + if bool(cache) == bool(snapshots): + raise ValueError('Choose exactly one cache or frozen snapshot directory') + if snapshots: + base = Path(snapshots).resolve() + matches = [r for r in json.loads((base/'receipts.json').read_text())['receipts'] if r['url'] == URL] + if len(matches) != 1: + raise ValueError('Expected one official program receipt') + r = matches[0]; filename = Path(r['file']) + if (filename.is_absolute() or '..' in filename.parts or + not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200): + raise ValueError('Unsafe or unsuccessful frozen program response') + data = (base/filename).read_bytes() + if len(data) > LIMIT or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']: + raise ValueError('Frozen program checksum/size mismatch') + observed = r['finished_at'] + else: + data = fetch(URL, cache, LIMIT, delay=1) + r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text()) + observed = r['retrieved_at'] + timestamp(observed) + return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data), + 'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.json', + 'observed_at': observed, 'http_status': 200, 'response_receipt': r} + + +def collect(out, *, cache=None, snapshots=None): + out = Path(out); out.mkdir(parents=True, exist_ok=True) + report_name = 'neurips-program-2026.json' + with (out/'neurips-program.collect.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if (out/report_name).exists(): + raise ValueError('Completed collection exists; use a new directory and reconcile before refresh') + data, evidence = read_program(cache, snapshots) + records, held, duplicates, folded = parse_program(data, evidence) + groups = defaultdict(list) + for r in records: + groups[r['track']].append(r) + files = [] + with tempfile.TemporaryDirectory(prefix='.neurips-program-', dir=out) as temporary: + stage = Path(temporary); (stage/'evidence').mkdir() + (stage/evidence['file']).write_bytes(data) + for track, group in sorted(groups.items()): + p = stage/f'neurips-program-2026-{track}.jsonl' + p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group)) + manifest = {'source': 'neurips_program', 'parser_version': 1, 'venue': 'NeurIPS', + 'year': 2026, 'track': track, 'count': len(group), 'file': str(out/p.name), + 'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL, + 'status': 'partial', 'missing_abstract_count': len(group), + 'note': 'Reviewed forum-level program partition; not full current proceedings, abstracts, public rejection coverage or scientific-work dedup.'} + p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n') + files.append(manifest) + result = {'source': 'neurips_program', 'year': 2026, 'finished_at': timestamp(), + 'status': 'collected_pending_identity_review', 'records': len(records), 'files': files, + 'held_presentations': held, 'exact_duplicate_event_rows': duplicates, + 'folded_extra_presentation_rows': folded, 'source_receipts': [evidence], + 'database_written': False, 'pdfs_downloaded': False, + 'note': 'Public program only; rejected opt-in availability needs a separate source. Journals, bridging and missing origins remain held; absence is not rejection.'} + (stage/report_name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') + (out/'evidence').mkdir(exist_ok=True) + for p in (stage/'evidence').iterdir(): + p.replace(out/'evidence'/p.name) + for p in stage.iterdir(): + if p.is_file() and p.name != report_name: + p.replace(out/p.name) + (stage/report_name).replace(out/report_name) + return result + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog) + parser.add_argument('--out', required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument('--cache') + source.add_argument('--snapshots') + args = parser.parse_args(argv) + result = collect(args.out, cache=args.cache, snapshots=args.snapshots) + print(json.dumps({'source': result['source'], 'year': 2026, 'status': result['status'], + 'records': result['records'], 'files': result['files'], + 'held_presentations': len(result['held_presentations']), + 'exact_duplicate_event_rows': len(result['exact_duplicate_event_rows']), + 'folded_extra_presentation_rows': result['folded_extra_presentation_rows'], + 'report_file': str(Path(args.out)/'neurips-program-2026.json'), + 'database_written': False, 'pdfs_downloaded': False}, ensure_ascii=False)) diff --git a/tests/fixtures/neurips2026-program.json.gz b/tests/fixtures/neurips2026-program.json.gz new file mode 100644 index 0000000..d4e419f Binary files /dev/null and b/tests/fixtures/neurips2026-program.json.gz differ diff --git a/tests/fixtures/neurips2026-receipts.json b/tests/fixtures/neurips2026-receipts.json new file mode 100644 index 0000000..dc5f1f6 --- /dev/null +++ b/tests/fixtures/neurips2026-receipts.json @@ -0,0 +1,15 @@ +{ + "receipts": [ + { + "url": "https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json", + "final_url": "https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json", + "status": 200, + "content_type": "application/json", + "file": "program-json.response", + "bytes": 20447021, + "sha256": "879293a863057fa05370443b3b5ddee12a10eeb37c810abc6c1d53f71f545997", + "started_at": "2026-11-09T17:57:23.825509+00:00", + "finished_at": "2026-11-09T17:57:40.125580+00:00" + } + ] +} diff --git a/tests/test_neurips_program.py b/tests/test_neurips_program.py new file mode 100644 index 0000000..af1ffa0 --- /dev/null +++ b/tests/test_neurips_program.py @@ -0,0 +1,174 @@ +import copy +from collections import Counter +import gzip +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from urllib.parse import parse_qs, urlsplit + +from paper_atlas.collectors import neurips_program as program + +FIXTURES = Path(__file__).parent/'fixtures' +RAW = gzip.decompress((FIXTURES/'neurips2026-program.json.gz').read_bytes()) +RECEIPTS = json.loads((FIXTURES/'neurips2026-receipts.json').read_text()) +EVIDENCE = {'url': program.URL, 'observed_at': RECEIPTS['receipts'][0]['finished_at']} + + +class NeuripsProgramTests(unittest.TestCase): + def sample(self): + rows = json.loads(RAW)['results'] + oral = next(r for r in rows if r['id'] == 154866) + poster = next(r for r in rows if r['paper_url'] == oral['paper_url'] and r['event_type'] == 'Poster') + return copy.deepcopy(poster), copy.deepcopy(oral) + + def payload(self, rows): + return json.dumps({'count': len(rows), 'next': None, 'previous': None, 'results': rows}) + + def snapshots(self, folder): + folder.mkdir() + (folder/'program-json.response').write_bytes(RAW) + (folder/'receipts.json').write_text(json.dumps(RECEIPTS)) + return folder + + def test_full_snapshot_tracks_and_all_titles_authors_affiliations(self): + rows, held, duplicates, folded = program.parse_program(RAW, EVIDENCE) + self.assertEqual((len(rows), len(held), len(duplicates), folded), (9004, 139, 1, 132)) + self.assertEqual(Counter(r['track'] for r in rows), + {'main': 7896, 'evaluations_datasets': 971, 'position': 137}) + expected = {} + for e in json.loads(RAW)['results']: + if e['sourceurl'] in program.GROUPS and e['event_type'] == 'Poster': + forum = parse_qs(urlsplit(e['paper_url']).query)['id'][0] + expected[forum] = e + self.assertEqual(len(expected), 9004) + for r in rows: + e = expected[r['identifiers']['openreview']] + self.assertEqual(r['title'], e['name']) + self.assertEqual(r['authors'], [a['fullname'] for a in e['authors']]) + self.assertEqual(r['metadata']['author_affiliations'], [a.get('institution') for a in e['authors']]) + self.assertEqual(r['raw_decision'], e['decision']) + self.assertEqual(r['status'], 'accepted') + self.assertNotIn('decision_date', r) + self.assertIsNone(r['abstract']) + self.assertEqual(r['year'], 2026) + self.assertEqual(duplicates[0]['event_id'], 154866) + + def test_same_forum_folds_presentations_and_preserves_signed_source_ids(self): + poster, oral = self.sample() + rows, held, duplicates, folded = program.parse_program(self.payload([oral, poster, oral]), EVIDENCE) + row, = rows + self.assertEqual((len(held), len(duplicates), folded), (0, 1, 1)) + self.assertEqual({p['sourceid'] for p in row['metadata']['presentations']}, + {poster['sourceid'], oral['sourceid']}) + self.assertLess(oral['sourceid'], 0) + self.assertEqual({t['value'] for t in row['tags']}, {'Poster', 'Oral'}) + self.assertTrue(all(t['namespace'] == 'neurips_program:presentation_format' for t in row['tags'])) + + def test_identical_title_and_authors_in_distinct_forums_remain_distinct(self): + poster, _ = self.sample(); other = copy.deepcopy(poster) + other.update(id=999999, uid='0'*32, sourceid=999999, paper_url='https://openreview.net/forum?id=otherForum7', + virtualsite_url='/virtual/2026/poster/999999') + rows, _, _, _ = program.parse_program(self.payload([poster, other]), EVIDENCE) + self.assertEqual(len(rows), 2) + self.assertEqual(len({r['external_id'] for r in rows}), 2) + + def test_conflicting_event_ids_or_same_forum_bibliography_are_rejected(self): + poster, oral = self.sample() + for field in ('name', 'authors', 'sourceurl', 'decision'): + changed = copy.deepcopy(oral) + if field == 'authors': + changed['authors'][0]['fullname'] = 'Different Author' + elif field == 'sourceurl': + changed[field] = 'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track' + elif field == 'decision': + changed[field] = 'Accept (poster)' + else: + changed[field] += ' changed' + with self.subTest(field=field), self.assertRaises(ValueError): + program.parse_program(self.payload([poster, changed]), EVIDENCE) + changed = copy.deepcopy(poster); changed['name'] += ' changed' + with self.assertRaises(ValueError): + program.parse_program(self.payload([poster, changed]), EVIDENCE) + + def test_unreviewed_origin_decision_visibility_and_link_are_held(self): + poster, _ = self.sample() + mutations = [{'sourceurl': 'journal-track/NeurIPS_2026-Posters_TMLR.xlsx'}, + {'sourceurl': 'neurips2026-bridging-the-future'}, + {'sourceurl': None}, {'sourceurl': 'https://openreview.net/group?id=NeurIPS.cc/2027/Conference'}, + {'decision': 'Reject'}, {'visible': False}, + {'paper_url': 'https://other.example/forum?id=anotherID7'}] + for mutation in mutations: + changed = copy.deepcopy(poster); changed.update(id=999999, virtualsite_url='/virtual/2026/poster/999999') + changed.update(mutation) + rows, held, _, _ = program.parse_program(self.payload([poster, changed]), EVIDENCE) + self.assertEqual((len(rows), len(held)), (1, 1)) + self.assertEqual(held[0]['raw_event'], changed) + + def test_rejects_truncation_pagination_invalid_url_and_missing_authors(self): + poster, _ = self.sample() + raw = json.loads(self.payload([poster])) + bad = [dict(raw, count=2), dict(raw, next='https://example.com/next'), + {k: v for k, v in raw.items() if k != 'previous'}] + for payload in bad: + with self.assertRaises(ValueError): + program.parse_program(json.dumps(payload), EVIDENCE) + for mutation in [{'authors': []}, {'virtualsite_url': '/virtual/2025/poster/154865'}, {'id': True}]: + changed = copy.deepcopy(poster); changed.update(mutation) + with self.assertRaises(ValueError): + program.parse_program(self.payload([changed]), EVIDENCE) + + def test_frozen_collection_is_partial_and_no_http_or_database_write(self): + with tempfile.TemporaryDirectory() as tmp, patch.object(program, 'fetch', side_effect=AssertionError('No HTTP')): + root = Path(tmp); source = self.snapshots(root/'source') + report = program.collect(root/'out', snapshots=source) + self.assertEqual(report['records'], 9004) + self.assertEqual(len(report['files']), 3) + self.assertFalse(report['database_written']) + self.assertFalse(report['pdfs_downloaded']) + for m in report['files']: + self.assertEqual(m['status'], 'partial') + self.assertEqual(m['missing_abstract_count'], m['count']) + self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256']) + with self.assertRaises(ValueError): + program.collect(root/'out', snapshots=source) + + def test_frozen_receipt_tamper_path_symlink_status_and_duplicate(self): + for mode in ('tamper', 'path', 'symlink', 'status', 'duplicate'): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); source = self.snapshots(root/'source') + receipts = copy.deepcopy(RECEIPTS) + if mode == 'tamper': + (source/'program-json.response').write_text('bad') + elif mode == 'path': + receipts['receipts'][0]['file'] = '../outside.json' + elif mode == 'symlink': + (root/'outside.json').write_bytes(RAW) + (source/'program-json.response').unlink() + (source/'program-json.response').symlink_to(root/'outside.json') + elif mode == 'status': + receipts['receipts'][0]['status'] = 403 + else: + receipts['receipts'].append(copy.deepcopy(receipts['receipts'][0])) + (source/'receipts.json').write_text(json.dumps(receipts)) + with self.assertRaises(ValueError): + program.collect(root/'out', snapshots=source) + self.assertFalse((root/'out/neurips-program-2026.json').exists()) + + def test_online_mode_only_requests_fixed_program_metadata(self): + poster, _ = self.sample(); raw = self.payload([poster]).encode() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); cache = root/'cache'; cache.mkdir() + receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': program.URL} + (cache/hashlib.sha256(program.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt)) + with patch.object(program, 'fetch', return_value=raw) as fetch: + report = program.collect(root/'out', cache=cache) + fetch.assert_called_once_with(program.URL, cache, program.LIMIT, delay=1) + self.assertEqual(report['records'], 1) + + def test_frozen_fixture_preserves_actual_response_bytes(self): + receipt, = RECEIPTS['receipts'] + self.assertEqual(hashlib.sha256(RAW).hexdigest(), receipt['sha256']) + self.assertEqual(len(RAW), receipt['bytes'])