Merge current main and preserve scope and AAMAS documentation entries

.gitea/workflows/check.yml

@@ -50,18 +50,18 @@ jobs:
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
-              subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
-              neurips_snapshots = root/'neurips2026-program-snapshots'
-              neurips_snapshots.mkdir()
-              (neurips_snapshots/'program-json.response').write_bytes(gzip.decompress((src/'tests/fixtures/neurips2026-program.json.gz').read_bytes()))
-              (neurips_snapshots/'receipts.json').write_bytes((src/'tests/fixtures/neurips2026-receipts.json').read_bytes())
-              subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--snapshots',str(neurips_snapshots),'--out',str(root/'installed-neurips2026-program')],cwd=outside,check=True)
-              neurips_report = json.loads((root/'installed-neurips2026-program/neurips-program-2026.json').read_text())
-              assert neurips_report['records'] == 9004 and len(neurips_report['held_presentations']) == 139
-              assert len(neurips_report['exact_duplicate_event_rows']) == 1 and neurips_report['folded_extra_presentation_rows'] == 132
-              assert not neurips_report['database_written'] and not neurips_report['pdfs_downloaded']
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
+              aamas2026 = json.loads(gzip.decompress((src/'tests/fixtures/aamas2026-fixture.json.gz').read_bytes()))
+              aamas2026_snapshots = root/'aamas2026-snapshots'
+              aamas2026_snapshots.mkdir()
+              (aamas2026_snapshots/'contents.html').write_text(aamas2026['contents'])
+              (aamas2026_snapshots/'receipts.json').write_text(json.dumps(aamas2026['receipts']))
+              subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2026','--snapshots',str(aamas2026_snapshots),'--out',str(root/'installed-aamas2026')],cwd=outside,check=True)
+              aamas2026_report = json.loads((root/'installed-aamas2026/aamas-proceedings-2026.json').read_text())
+              assert aamas2026_report['records'] == 639 and len(aamas2026_report['held_publications']) == 1
+              assert aamas2026_report['status_counts'] == {'accepted': 638, 'withdrawn': 1}
+              assert not aamas2026_report['database_written'] and not aamas2026_report['pdfs_downloaded']
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)

README.md

@@ -8,6 +8,8 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追
 
 当前主动补漏从 **2020 年**起；2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md)，下次从这里接续。已有旧数据保留，范围排除不算补齐。
 
+AAMAS 2026 的官方 HTML 目录由 `collect aamas-proceedings --year 2026` 适配，区分 track 和发表形式，keynote 排除、撤稿证据单独保留；[采集与恢复说明](docs/aamas-proceedings.md)。采集不写数据库，仍需身份核验和增量导入。
+
 历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配，保留主会/workshop及重复展示证据；运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
 
 `collect springer-selected` 核验限定的会后选集，分开保存会议年份、出版年份及选集事件身份。目前支持 AISTATS 1993 对应的 1994 年出版选集，原始会议决定仍须单独核查；配置、证据和恢复见 [会后选集说明](docs/springer-selected-volumes.md)。
@@ -68,8 +70,6 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
   --out /data/paper-index/collected
 ```
 
-NeurIPS 2026 的[官方节目采集器](docs/neurips-program.md)按 main、evaluations/datasets、position 分开保存，合并同一 forum 的口头与海报展示，保留期刊等未决关系；运行 `paper-atlas collect neurips-program`。它只生成元数据文件，身份核验和生产导入另行执行。
-
 早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别，保留原名与范围证据；代码支持不代表已补齐生产数据。
 
 EWSL、EPS、ECP、EWSP 的[具体前身卷与完整快照要求](docs/coverage/historical-predecessors-2026-11-09.md)单独核验，保留原始系列及事件/出版年份；旧候选文件缺少 ECP/EWSP 时会明确提示，避免静默遗漏。

docs/aamas-proceedings.md

@@ -1,6 +1,31 @@
 # AAMAS 官方论文集元数据
 
-`paper_atlas.collectors.aamas` 是按需运行的采集适配器，通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025** HTML 版式；其他年份需先核对官方目录和逐篇结构，再增加适配，不能仅替换年份。2025 解析单独封装在 `paper_atlas.collectors.aamas2025`。
+`paper_atlas.collectors.aamas` 是按需运行的采集适配器，通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025、2026** HTML 版式；其他年份需先核对官方目录和逐篇结构，再增加适配，不能仅替换年份。2025、2026 解析分别封装在 `paper_atlas.collectors.aamas2025`、`paper_atlas.collectors.aamas2026`。
+
+
+## AAMAS 2026
+
+2026 只读取[官方 HTML 目录](https://www.ifaamas.org/Proceedings/aamas2026/forms/contents.htm)，不下载 PDF。冻结目录 SHA256 `ce2bc7fc5da340930343901506a9ce37c0334d4068f07546431c9deee3f13de9`，643 个条目：keynote 3、research 338、extended abstract 185、AAAI 19、Blue Sky 17、doctoral 34、demo 44、JAAMAS 3。
+
+排除 3 个 keynote；research 的 p238 明确标注 `Retracted June 8, 2026`，完整题名、作者、栏目、日期、原文和来源回执保存在采集报告 `held_publications` 中，**不会输出 accepted 或 withdrawn 投稿记录**。当前决定模型没有独立 retracted 状态，后续须单独建模和核验；原始证据不能删除。任何新增或改变的撤稿标记都会停止采集，等待人工核查。
+
+另一个 extended abstract p3178 明确标注 `Withdrawn June 8, 2026`，输出 `withdrawn`，在 metadata 保留日期 `2026-06-08` 和日精度；原网页没有时区/时刻，不能伪造决定 timestamp，不补造其此前的接受决定或 PDF 链接。因此 639 个输出条目是 **638 accepted、1 withdrawn**，按六个 track 分组：`proceedings_unspecified` 522、`aaai_transfer` 19、`blue_sky` 17、`doctoral` 34、`demo` 44、`jaamas` 3。全文/扩展摘要是发表形式，不是研究类型；research 栏目不能证明每篇原始 main/special 投稿轨道。JAAMAS 保留期刊展示语义，不制造另一次独立审稿录用。
+
+[AAAI 转投详细政策](https://cyprusconferences.org/aamas2026/call-for-papers-aaai-track/) 要求基于 AAAI-26 rejected 稿件；[总览](https://cyprusconferences.org/aamas2026/calls/) 却写 AAAI-25，须保留冲突。采集器保留 AAAI 栏目归属，不据此生成原 AAAI 投稿 ID、年份或逐篇拒稿决定。目录不是原始决定 note，未来关联仍需逐篇证据。
+
+```sh
+paper-atlas collect aamas-proceedings --year 2026 \
+  --snapshots /data/paper-index/collected/aamas2026-primary-evidence-v1 \
+  --out /data/paper-index/collected/aamas2026-normalized-v1
+```
+
+在线模式将 `--snapshots` 换成 `--cache /data/paper-index/cache/aamas2026`。冻结模式 `receipts.json` 的 `receipts` 至少包含官方目录 URL、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区 `finished_at`。拒绝越界路径、越界符号链接、失败状态、哈希/大小不符的页面。这里只读取目录对应回执，其他已冻结政策网页继续保留。
+
+输出六组 JSONL/manifest、原始 HTML 与最后发布的 `aamas-proceedings-2026.json`；统一入口沿用输出目录锁和暂存目录。manifest 全部为 `partial`，639 条 HTML 摘要缺失如实记录。发布编号使用年份和起始页，随机 PDF 文档 ID 单独保留，不能沿用 2025 的 pN.pdf 规则。
+
+首次部署先核对来源哈希和生产状态，再在新输出目录运行采集、独立比对 643 条标题/作者/栏目分区，随后做 PostgreSQL 身份预检与增量导入。采集、远程 CI 通过或目录含某个年份均不代表生产已导入、摘要齐全或最终去重完成。较早年份不自动启动补采，当前主动补漏范围按用户要求从 2020 年起。
+
+## AAMAS 2025
 
 2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条：3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote，输出 476 条；Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告，发表年份仍为 2025，格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`，保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。
 

docs/neurips-program.md

@@ -1,56 +0,0 @@
-# NeurIPS 官方节目元数据
-
-`paper-atlas collect neurips-program` 是按需运行的采集器，当前核验范围是 2026 年的[官方 orals/posters JSON](https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json)。它只请求这个固定公开元数据文件，不连接数据库，不请求 OpenReview API，不下载 PDF、视频或个人资料页。模块位于 `paper_atlas.collectors.neurips_program`，通过统一 CLI 使用。
-
-## 已核验快照
-
-2026-11-09T17:57:40Z 冻结的原始响应共 20,447,021 字节，SHA256 `879293a863057fa05370443b3b5ddee12a10eeb37c810abc6c1d53f71f545997`。原始 `count=9276` 与结果数组长度一致，`next`、`previous` 均为空；这只证明此份公开文件已完整取得，不是全部会议出版物或公开拒稿全集。
-
-按官方逐条 `sourceurl` 中的明确投稿来源分开处理：
-
-| 原始 OpenReview group | 数据库 track | 不同 forum 数 |
-| --- | --- | ---: |
-| `NeurIPS.cc/2026/Conference` | `main` | 7896 |
-| `NeurIPS.cc/2026/Evaluations_and_Datasets_Track` | `evaluations_datasets` | 971 |
-| `NeurIPS.cc/2026/Position_Paper_Track` | `position` | 137 |
-| 合计 | | 9004 |
-
-只接受已核验的显式 `Accept`、`Accept (poster/spotlight/oral)` 标签，保留原始大小写和文本。未知来源、决定、展示类型、可见性或无法验证的 forum 链接进入 `held_presentations`，不推成 accepted 或 rejected。空标题、缺少作者、相同 event ID 不同内容、同一 forum 的题名/作者/来源/决定冲突会停止采集。
-
-原 9276 行中，event 154866 有一条内容完全相同的重复行，单独保留其原始行号；另外 132 个口头展示与同 forum 的海报合并为一条投稿候选，完整保留两种展示及时间、场次、原始带符号 `sourceid`。负数 `sourceid` 不能被解释为拒稿或另一次投稿，也不转换为全局论文标识。
-
-其余 139 行单独留档：87 个期刊展示、36 个 Bridging the Future 展示、16 个未提供明确投稿来源的条目。这些不因缺少链接而被推为拒稿，也不能被冒充为独立主会审稿录用。后续须核实原期刊发表、认证、复现或其他展示关系。
-
-9004 条是不同 forum 的来源记录，不是已经新增 9004 篇独立科学论文。不同 forum 即使题名和作者相同也保留各自投稿；跨会及版本关系另行核验。题名、完整作者顺序和机构取自原始 JSON，口头与海报不是研究类型；展示标签独立使用 `neurips_program:presentation_format` namespace。节目中的未来展示时间不冒充决定时间。
-
-## 运行与恢复
-
-在线模式只请求一个固定元数据 URL，并复用受限大小的响应缓存：
-
-```sh
-paper-atlas collect neurips-program \
-  --cache /data/paper-index/cache/neurips2026-program \
-  --out /data/paper-index/collected/neurips2026-program-YYYYMMDD
-```
-
-冻结模式只读 `receipts.json` 对应的原始文件：
-
-```sh
-paper-atlas collect neurips-program \
-  --snapshots /data/paper-index/collected/neurips2026-program-probe-v1 \
-  --out /data/paper-index/collected/neurips2026-program-normalized-v1
-```
-
-快照回执包含精确 `url`、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区的 `finished_at`。拒绝越界路径/符号链接、重复回执、失败响应、大小或哈希不符。在线模式沿用项目 HTTP 缓存和限速，验证回执时间后保存原始 JSON。输出目录有互斥锁，完成报告最后发布；已有完成报告时要求新建输出目录，避免覆盖后误重放。
-
-输出三个 track 的 JSONL/manifest、原始 JSON 证据和 `neurips-program-2026.json`。manifest 始终为 `partial`；原始节目 JSON 没有摘要，9004 条 `abstract=null` 和缺失原因如实保存，不能用别篇摘要填充。命令终端只输出数量与路径，完整待处理展示与原始行号在报告中。
-
-恢复时先校验文件和回执，使用同一完整快照生成新输出；若已有完成报告或导入回执，先核实实际来源绑定和 checkpoint，不重放已消费队列。生成文件后还需 PostgreSQL 身份预检、关联审核、增量队列、原生导入和独立验收；源文件成功、CI 通过均不能替代生产导入或最终全库去重。
-
-## 目录差异与公开拒稿限制
-
-同期[官网 Downloads 页面](https://neurips.cc/Downloads/2026)的活动总数不是本 JSON 的论文总数。逐个 poster ID/题名对照后，网页另有 event 155165（Lemon）未出现在本 JSON 中，须单独取得并审核元数据，不能补造其 track 或 forum。16 个未提供来源的条目在 JSON 中使用通用 session URL，而网页有各自 poster URL；原值保留，不能因 URL 映射差异制造新论文。
-
-[2026 Main Track Handbook](https://neurips.cc/Conferences/2026/MainTrackHandbook)明确允许 rejected 作者选择公开。当前无账号公开 notes API 请求返回 `ChallengeRequiredError`、HTTP 403，失败原文和回执保留，不绕过验证；因此不能把本 accepted 节目文件当作拒稿数据源或据此推断拒稿是否存在。官网下载表单的 JSON 导出也曾超时，未取得导出内容，已改用独立公开静态节目文件并保留失败证据，不重试超时导出。
-
-当前主动补漏范围从 2020 年起。期刊/展示关系、网页独有条目、摘要和公开负面决定仍有待办；冻结资料和代码适配不代表完整会议覆盖。

paper_atlas/cli.py

@@ -26,7 +26,6 @@ COMMANDS = {
     ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
     ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
     ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
-    ('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
     ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
     ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
     ('collect','dblp-family'): ('paper_atlas.collectors.dblp_recovery', {}, '从固定快照恢复指定 DBLP family 的原始候选'),

paper_atlas/collectors/aamas.py

@@ -136,6 +136,9 @@ def read_pages(cache=None, snapshots=None):
 
 
 def _collect(year, out, *, cache=None, snapshots=None):
+    if year == 2026:
+        from paper_atlas.collectors.aamas2026 import collect as collect_2026
+        return collect_2026(out, cache=cache, snapshots=snapshots)
     if year == 2025:
         from paper_atlas.collectors.aamas2025 import collect as collect_2025
         return collect_2025(out, cache=cache, snapshots=snapshots)
@@ -193,8 +196,8 @@ def _collect(year, out, *, cache=None, snapshots=None):
 
 
 def collect(year, out, *, cache=None, snapshots=None):
-    if year not in (2012, 2025):
-        raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported')
+    if year not in (2012, 2025, 2026):
+        raise ValueError('Only independently reviewed AAMAS 2012/2025/2026 layouts are supported')
     out = Path(out)
     out.mkdir(parents=True, exist_ok=True)
     with (out/'aamas.collect.lock').open('a') as lock:
@@ -204,7 +207,7 @@ def collect(year, out, *, cache=None, snapshots=None):
 
 def main(argv=None, *, prog=None):
     parser = argparse.ArgumentParser(prog=prog)
-    parser.add_argument('--year', type=int, choices=[2012, 2025], required=True)
+    parser.add_argument('--year', type=int, choices=[2012, 2025, 2026], required=True)
     parser.add_argument('--out', required=True)
     source = parser.add_mutually_exclusive_group(required=True)
     source.add_argument('--cache')

paper_atlas/collectors/aamas2026.py

@@ -0,0 +1,219 @@
+"""Reviewed AAMAS 2026 HTML metadata; retractions stay outside accepted queues."""
+from collections import Counter, defaultdict
+import hashlib
+import json
+from pathlib import Path
+import re
+import tempfile
+
+from paper_atlas.collectors.aamas2025 import one
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.io.html import Document, Node, clean
+from paper_atlas.io.http import fetch
+
+ROOT = 'https://www.ifaamas.org/Proceedings/aamas2026/'
+URL = ROOT+'forms/contents.htm'
+TITLE = 'AAMAS 2026: Proceedings of the 2026 International Conference on Autonomous Agents and Multiagent Systems'
+SECTIONS = {
+    'K': ('Keynote Talks', 'keynote', 'keynote', 3),
+    'R': ('Research Paper Track', 'full_paper', 'proceedings_unspecified', 338),
+    '1': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 185),
+    'A': ('AAAI Track', 'full_paper', 'aaai_transfer', 19),
+    'B': ('Blue Sky Ideas Track', 'blue_sky_paper', 'blue_sky', 17),
+    'C': ('Doctoral Consortium', 'doctoral_abstract', 'doctoral', 34),
+    'D': ('Demonstration Track', 'demo', 'demo', 44),
+    'J': ('JAAMAS Track', 'journal_presentation', 'jaamas', 3),
+}
+PDF = re.compile(r'\.\./pdfs/([A-Z]{4}[0-9]{4})\.pdf')
+RETRACTION_TITLE = 'Unifying Model-Free Efficiency and Model-Based Representations via Latent Dynamics'
+WITHDRAWAL_TITLE = 'Cognitive Heterogeneity and Behavioral Biases in Multi-Stage Supply Chains: Evidence from LLM-Based Agent Simulations'
+
+
+def parse_authors(paragraph):
+    font = one([n for n in paragraph.walk() if n.tag == 'font' and
+                n.attrs.get('color', '').lower() == '#222222'], 'ordered author block')
+    authors, affiliations = [], []
+    name, affiliation = '', ''
+    def flush():
+        nonlocal name, affiliation
+        if clean(name) or clean(affiliation):
+            if not clean(name):
+                raise ValueError('Affiliation without author')
+            authors.append(clean(name)); affiliations.append(clean(affiliation))
+        name, affiliation = '', ''
+    for child in font.children:
+        if isinstance(child, str):
+            name += child
+        elif child.tag == 'br':
+            flush()
+        elif child.tag == 'i':
+            affiliation += child.text()
+        else:
+            raise ValueError('Unreviewed author markup: '+child.tag)
+    flush()
+    if not authors:
+        raise ValueError('Empty author list')
+    return authors, affiliations
+
+
+def parse_contents(html, evidence):
+    doc = Document(html)
+    if [clean(n.text()) for n in doc.root.walk() if n.tag == 'title'] != [TITLE]:
+        raise ValueError('Unexpected AAMAS proceedings title/year')
+    section = None
+    headers, records, excluded, held = [], [], [], []
+    counts, seen_pages, seen_pdf_ids = Counter(), set(), set()
+    for node in doc.root.walk():
+        if node.tag != 'p' or any(n is not node and n.tag == 'p' for n in node.walk()):
+            continue
+        anchors = [n.attrs['name'] for n in node.walk() if n.tag == 'a' and n.attrs.get('name') in SECTIONS]
+        if anchors:
+            section = one(anchors, 'section anchor')
+            if clean(node.text()) != SECTIONS[section][0]:
+                raise ValueError('Publisher section heading differs from reviewed layout')
+            headers.append(section)
+        page_fonts = [n for n in node.walk() if n.tag == 'font' and
+                      n.attrs.get('color', '').lower() == '#444444']
+        if not page_fonts:
+            continue
+        page_text = clean(one(page_fonts, 'page marker').text())
+        page_match = re.fullmatch(r'\(Page ([1-9][0-9]*)\)', page_text)
+        if not page_match or section not in SECTIONS:
+            raise ValueError('Invalid page or paper outside reviewed sections')
+        page = int(page_match[1])
+        if page in seen_pages:
+            raise ValueError('Repeated publisher page ID')
+        seen_pages.add(page)
+        title_node = one([n for n in node.walk() if n.tag == 'strong'], 'paper title')
+        title = clean(title_node.text())
+        if not title:
+            raise ValueError('Empty paper title')
+        authors, affiliations = parse_authors(node)
+        link = one([n for n in node.walk() if n.tag == 'a' and 'href' in n.attrs], 'paper link')
+        href = link.attrs['href']
+        title_nodes = {id(n) for n in title_node.walk()}
+        retractions = [clean(n.text()) for n in link.walk() if n.tag == 'i' and id(n) not in title_nodes]
+        heading, publication_format, track, _ = SECTIONS[section]
+        counts[section] += 1
+        metadata = {'publisher_page': page, 'section_url': URL, 'section_anchor': section,
+                    'section_heading': heading, 'publication_format': publication_format,
+                    'author_affiliations': affiliations, 'source_evidence': evidence,
+                    'decision_date_precision': 'not_supplied',
+                    'missing_abstract_reason': 'Publisher HTML supplies no abstract; research PDF not downloaded.',
+                    'publication_evidence': 'Published proceedings entry, not an original decision note.'}
+        withdrawn = retractions == ['Withdrawn June 8, 2026']
+        if withdrawn:
+            if (section, page, title, href) != ('1', 3178, WITHDRAWAL_TITLE, ''):
+                raise ValueError('Unreviewed withdrawal notice')
+            metadata.update({'publication_state': 'withdrawn',
+                             'withdrawal_date': '2026-06-08',
+                             'raw_publication_state': retractions[0],
+                             'decision_date_precision': 'day',
+                             'publication_evidence': 'Explicit dated publisher withdrawal notice, not an original review decision note.'})
+        elif retractions:
+            if (section, page, title, href, retractions) != ('R', 238, RETRACTION_TITLE, '', ['Retracted June 8, 2026']):
+                raise ValueError('Unreviewed retraction or publication status markup')
+            held.append({'source': 'aamas_proceedings', 'external_id': '2026/p238',
+                         'title': title, 'authors': authors, 'year': 2026, 'track': track,
+                         'url': URL+'#R', 'abstract': None, 'identifiers': {},
+                         'observed_at': evidence['observed_at'], 'metadata': metadata,
+                         'publication_state': 'retracted', 'retraction_date': '2026-06-08',
+                         'raw_publication_state': retractions[0],
+                         'hold_reason': 'Explicit retraction; not an accepted/withdrawn submission assertion. Dedicated decision modeling and identity review required.'})
+            continue
+        if not withdrawn:
+            match = PDF.fullmatch(href)
+            if not match or match[1] in seen_pdf_ids:
+                raise ValueError('Invalid or repeated publisher PDF ID')
+            seen_pdf_ids.add(match[1])
+            metadata.update({'publisher_document_id': match[1], 'paper_url': ROOT+'pdfs/'+match[1]+'.pdf'})
+        if section == 'K':
+            excluded.append({'publisher_page': page, 'title': title, 'authors': authors,
+                             'metadata': metadata, 'reason': 'Publisher keynote, not a research submission.'})
+            continue
+        metadata['track_scope_note'] = ('Published format known; individual main/special submission track not established.'
+                                        if track == 'proceedings_unspecified' else 'Explicit publisher section.')
+        if section == 'A':
+            metadata['prior_submission_note'] = 'AAAI transfer section does not supply original submission IDs or individual prior decision notes; none inferred.'
+        if section == 'J':
+            metadata['track_scope_note'] = 'Journal presentation; not another independent journal review or research-track acceptance.'
+        records.append({'source': 'aamas_proceedings', 'external_id': '2026/p'+str(page),
+                        'url': URL+'#'+section, 'title': title, 'abstract': None,
+                        'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
+                        'year': 2026, 'track': track, 'status': 'withdrawn' if withdrawn else 'accepted',
+                        'observed_at': evidence['observed_at'],
+                        'raw_decision': retractions[0] if withdrawn else 'listed in official AAMAS 2026 published proceedings '+heading,
+                        'metadata': metadata,
+                        'tags': [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]})
+    if headers != list(SECTIONS) or dict(counts) != {k: v[3] for k, v in SECTIONS.items()}:
+        raise ValueError('Publisher section inventory differs from reviewed 2026 layout')
+    if (len(records), len(excluded), len(held)) != (639, 3, 1):
+        raise ValueError('Unexpected accepted/keynote/retraction partition')
+    if Counter(r['status'] for r in records) != {'accepted': 638, 'withdrawn': 1}:
+        raise ValueError('Unexpected publisher publication-state partition')
+    return records, excluded, held
+
+
+def read_page(cache, snapshots):
+    if bool(cache) == bool(snapshots):
+        raise ValueError('Choose exactly one cache or frozen snapshot directory')
+    if snapshots:
+        base = Path(snapshots).resolve()
+        frozen = json.loads((base/'receipts.json').read_text())['receipts']
+        r = one([r for r in frozen if r['url'] == URL], 'frozen response receipt')
+        filename = Path(r['file'])
+        if (filename.is_absolute() or '..' in filename.parts or
+                not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200):
+            raise ValueError('Unsafe or unsuccessful publisher response')
+        data = (base/filename).read_bytes()
+        if len(data) > 8*1024**2 or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
+            raise ValueError('Frozen publisher response checksum/size mismatch')
+        observed = r['finished_at']
+    else:
+        data = fetch(URL, cache, 8*1024**2, delay=1)
+        r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text())
+        observed = r['retrieved_at']
+    timestamp(observed)
+    return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
+                  'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
+                  'observed_at': observed, 'http_status': 200, 'response_receipt': r}
+
+
+def collect(out, *, cache=None, snapshots=None):
+    data, evidence = read_page(cache, snapshots)
+    records, excluded, held = parse_contents(data.decode('utf-8'), evidence)
+    grouped = defaultdict(list)
+    for row in records:
+        grouped[row['track']].append(row)
+    out = Path(out); out.mkdir(parents=True, exist_ok=True); files = []
+    with tempfile.TemporaryDirectory(prefix='.aamas2026-', dir=out) as temporary:
+        stage = Path(temporary); (stage/'evidence').mkdir()
+        (stage/evidence['file']).write_bytes(data)
+        for track, group in sorted(grouped.items()):
+            p = stage/f'aamas-proceedings-2026-{track}.jsonl'
+            p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
+            manifest = {'source': 'aamas_proceedings', 'parser_version': 3, 'venue': 'AAMAS', 'year': 2026,
+                        'track': track, 'count': len(group), 'file': str(out/p.name),
+                        'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL,
+                        'status': 'partial', 'missing_abstract_count': len(group),
+                        'status_counts': dict(Counter(r['status'] for r in group)),
+                        'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
+                        'note': 'Reviewed TOC partition excluding keynote/retracted entries; identity and full submission history remain unverified.'}
+            p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
+            files.append(manifest)
+        result = {'source': 'aamas_proceedings', 'year': 2026, 'finished_at': timestamp(),
+                  'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
+                  'excluded_publications': excluded, 'held_publications': held,
+                  'status_counts': dict(Counter(r['status'] for r in records)),
+                  'source_receipts': [evidence], 'database_written': False, 'pdfs_downloaded': False,
+                  'note': 'Retraction evidence held separately; publication is not an original decision note, and absence is not rejection.'}
+        report = 'aamas-proceedings-2026.json'
+        (stage/report).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
+        (out/'evidence').mkdir(exist_ok=True)
+        for p in (stage/'evidence').iterdir():
+            p.replace(out/'evidence'/p.name)
+        for p in stage.iterdir():
+            if p.is_file() and p.name != report:
+                p.replace(out/p.name)
+        (stage/report).replace(out/report)
+    return result

paper_atlas/collectors/neurips_program.py

@@ -1,218 +0,0 @@
-"""Official NeurIPS program metadata, with presentations separate from submissions."""
-import argparse
-from collections import Counter, defaultdict
-import fcntl
-import hashlib
-import json
-from pathlib import Path
-import re
-import tempfile
-from urllib.parse import parse_qs, urlsplit
-
-from paper_atlas.db.submissions import timestamp
-from paper_atlas.io.http import fetch
-
-URL = 'https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json'
-GROUPS = {
-    'https://openreview.net/group?id=NeurIPS.cc/2026/Conference': 'main',
-    'https://openreview.net/group?id=NeurIPS.cc/2026/Evaluations_and_Datasets_Track': 'evaluations_datasets',
-    'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track': 'position',
-}
-ACCEPTED = {'Accept', 'Accept (poster)', 'Accept (spotlight)', 'Accept (oral)',
-            'Accept (Spotlight)', 'Accept (Oral)'}
-LIMIT = 64*1024**2
-
-
-def forum_id(url):
-    if not isinstance(url, str):
-        return None
-    try:
-        parsed = urlsplit(url)
-    except ValueError:
-        return None
-    query = parse_qs(parsed.query, keep_blank_values=True)
-    if (parsed.scheme != 'https' or parsed.netloc != 'openreview.net' or
-            parsed.path != '/forum' or parsed.fragment or set(query) != {'id'} or
-            len(query['id']) != 1 or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', query['id'][0])):
-        return None
-    return query['id'][0]
-
-
-def parse_program(data, evidence):
-    root = json.loads(data)
-    if (not isinstance(root, dict) or not {'count', 'next', 'previous', 'results'} <= set(root) or
-            not isinstance(root.get('results'), list) or
-            type(root.get('count')) is not int or root['count'] != len(root['results']) or
-            root.get('next') is not None or root.get('previous') is not None):
-        raise ValueError('Expected one complete, nonpaginated public program snapshot')
-    if not root['results']:
-        raise ValueError('Empty program snapshot')
-    events, grouped, held, duplicates = {}, defaultdict(list), [], []
-    for index, event in enumerate(root['results']):
-        if not isinstance(event, dict) or type(event.get('id')) is not int or event['id'] <= 0:
-            raise ValueError('Missing stable event ID')
-        eid = event['id']
-        if eid in events:
-            first_index, first = events[eid]
-            if first != event:
-                raise ValueError('Conflicting payloads share one publisher event ID')
-            duplicates.append({'event_id': eid, 'first_row': first_index, 'duplicate_row': index})
-            continue
-        events[eid] = index, event
-        origin = event.get('sourceurl')
-        reason = None
-        if not isinstance(origin, str) or origin not in GROUPS:
-            reason = ('journal_presentation_requires_original_publication_relation'
-                      if isinstance(origin, str) and origin.startswith('journal-track/') else
-                      'bridging_future_scope_requires_review' if origin == 'neurips2026-bridging-the-future' else
-                      'unreviewed_or_missing_submission_origin')
-        elif event.get('visible') is not True:
-            reason = 'program_visibility_requires_review'
-        elif not isinstance(event.get('decision'), str) or event['decision'] not in ACCEPTED:
-            reason = 'unreviewed_decision_label'
-        elif event.get('event_type') not in ('Poster', 'Oral'):
-            reason = 'unreviewed_presentation_type'
-        forum = forum_id(event.get('paper_url'))
-        if reason is None and forum is None:
-            reason = 'missing_or_invalid_original_submission_link'
-        if reason:
-            held.append({'row': index, 'event_id': eid, 'reason': reason, 'raw_event': event})
-            continue
-        if (not isinstance(event.get('name'), str) or not event['name'].strip() or
-                not isinstance(event.get('authors'), list) or not event['authors'] or
-                any(not isinstance(a, dict) or not isinstance(a.get('fullname'), str) or
-                    not a['fullname'].strip() for a in event['authors'])):
-            raise ValueError('Empty title or incomplete ordered author list')
-        if not isinstance(event.get('virtualsite_url'), str) or not re.fullmatch(
-                r'/virtual/2026/(?:poster|oral)/'+str(eid), event['virtualsite_url']):
-            raise ValueError('Publisher event URL differs from year/type/ID')
-        grouped[forum].append((index, event))
-    records = []
-    folded = 0
-    for forum, observations in sorted(grouped.items()):
-        # Signed sourceid values are preserved verbatim in the presentation
-        # evidence; they are not independent submissions or scientific works.
-        signatures = {json.dumps([e['name'], e['authors'], e['sourceurl'], e['decision']],
-                                 ensure_ascii=False, sort_keys=True) for _, e in observations}
-        if len(signatures) != 1:
-            raise ValueError('Title, authors, origin or decision disagree for one forum')
-        posters = [e for _, e in observations if e['event_type'] == 'Poster']
-        event = posters[0] if posters else observations[0][1]
-        track = GROUPS[event['sourceurl']]
-        presentations = []
-        for index, e in observations:
-            presentations.append({'row': index, 'event_id': e['id'], 'uid': e.get('uid'),
-                                  'sourceid': e.get('sourceid'), 'event_type': e['event_type'],
-                                  'eventtype': e.get('eventtype'), 'program_url': e['virtualsite_url'],
-                                  'session': e.get('session'), 'starttime': e.get('starttime'),
-                                  'endtime': e.get('endtime'), 'parent_id': e.get('parent_id'),
-                                  'raw_decision': e['decision']})
-        formats = sorted({e['event_type'] for _, e in observations})
-        records.append({
-            'source': 'neurips_program', 'external_id': '2026/'+forum,
-            'url': event['paper_url'], 'title': event['name'], 'abstract': None,
-            'authors': [a['fullname'] for a in event['authors']],
-            'identifiers': {'openreview': forum}, 'venue': 'NeurIPS', 'venue_aliases': [],
-            'year': 2026, 'track': track, 'status': 'accepted',
-            'observed_at': evidence['observed_at'], 'raw_decision': event['decision'],
-            'metadata': {'source_group_url': event['sourceurl'], 'forum_id': forum,
-                         'author_affiliations': [a.get('institution') for a in event['authors']],
-                         'presentations': presentations, 'source_evidence': evidence,
-                         'publication_evidence': 'Explicit accepted label in official conference program; not an original decision note or proof of proceedings publication.',
-                         'decision_date_precision': 'not_supplied',
-                         'missing_abstract_reason': 'Program JSON does not contain abstracts; paper HTML/PDF not fetched by this command.',
-                         'identity_scope_note': 'One record per original forum, not one scientific work per event. Other venues/versions require independent identity review.'},
-            'tags': [{'namespace': 'neurips_program:presentation_format', 'value': value}
-                     for value in formats]})
-        folded += len(observations)-1
-    if not records:
-        raise ValueError('No reviewed submission origins found')
-    if root['count'] != len(records)+len(held)+len(duplicates)+folded:
-        raise ValueError('Program partition failed')
-    return records, held, duplicates, folded
-
-
-def read_program(cache, snapshots):
-    if bool(cache) == bool(snapshots):
-        raise ValueError('Choose exactly one cache or frozen snapshot directory')
-    if snapshots:
-        base = Path(snapshots).resolve()
-        matches = [r for r in json.loads((base/'receipts.json').read_text())['receipts'] if r['url'] == URL]
-        if len(matches) != 1:
-            raise ValueError('Expected one official program receipt')
-        r = matches[0]; filename = Path(r['file'])
-        if (filename.is_absolute() or '..' in filename.parts or
-                not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200):
-            raise ValueError('Unsafe or unsuccessful frozen program response')
-        data = (base/filename).read_bytes()
-        if len(data) > LIMIT or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
-            raise ValueError('Frozen program checksum/size mismatch')
-        observed = r['finished_at']
-    else:
-        data = fetch(URL, cache, LIMIT, delay=1)
-        r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text())
-        observed = r['retrieved_at']
-    timestamp(observed)
-    return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
-                  'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.json',
-                  'observed_at': observed, 'http_status': 200, 'response_receipt': r}
-
-
-def collect(out, *, cache=None, snapshots=None):
-    out = Path(out); out.mkdir(parents=True, exist_ok=True)
-    report_name = 'neurips-program-2026.json'
-    with (out/'neurips-program.collect.lock').open('a') as lock:
-        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        if (out/report_name).exists():
-            raise ValueError('Completed collection exists; use a new directory and reconcile before refresh')
-        data, evidence = read_program(cache, snapshots)
-        records, held, duplicates, folded = parse_program(data, evidence)
-        groups = defaultdict(list)
-        for r in records:
-            groups[r['track']].append(r)
-        files = []
-        with tempfile.TemporaryDirectory(prefix='.neurips-program-', dir=out) as temporary:
-            stage = Path(temporary); (stage/'evidence').mkdir()
-            (stage/evidence['file']).write_bytes(data)
-            for track, group in sorted(groups.items()):
-                p = stage/f'neurips-program-2026-{track}.jsonl'
-                p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
-                manifest = {'source': 'neurips_program', 'parser_version': 1, 'venue': 'NeurIPS',
-                            'year': 2026, 'track': track, 'count': len(group), 'file': str(out/p.name),
-                            'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL,
-                            'status': 'partial', 'missing_abstract_count': len(group),
-                            'note': 'Reviewed forum-level program partition; not full current proceedings, abstracts, public rejection coverage or scientific-work dedup.'}
-                p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
-                files.append(manifest)
-            result = {'source': 'neurips_program', 'year': 2026, 'finished_at': timestamp(),
-                      'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
-                      'held_presentations': held, 'exact_duplicate_event_rows': duplicates,
-                      'folded_extra_presentation_rows': folded, 'source_receipts': [evidence],
-                      'database_written': False, 'pdfs_downloaded': False,
-                      'note': 'Public program only; rejected opt-in availability needs a separate source. Journals, bridging and missing origins remain held; absence is not rejection.'}
-            (stage/report_name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
-            (out/'evidence').mkdir(exist_ok=True)
-            for p in (stage/'evidence').iterdir():
-                p.replace(out/'evidence'/p.name)
-            for p in stage.iterdir():
-                if p.is_file() and p.name != report_name:
-                    p.replace(out/p.name)
-            (stage/report_name).replace(out/report_name)
-        return result
-
-
-def main(argv=None, *, prog=None):
-    parser = argparse.ArgumentParser(prog=prog)
-    parser.add_argument('--out', required=True)
-    source = parser.add_mutually_exclusive_group(required=True)
-    source.add_argument('--cache')
-    source.add_argument('--snapshots')
-    args = parser.parse_args(argv)
-    result = collect(args.out, cache=args.cache, snapshots=args.snapshots)
-    print(json.dumps({'source': result['source'], 'year': 2026, 'status': result['status'],
-                      'records': result['records'], 'files': result['files'],
-                      'held_presentations': len(result['held_presentations']),
-                      'exact_duplicate_event_rows': len(result['exact_duplicate_event_rows']),
-                      'folded_extra_presentation_rows': result['folded_extra_presentation_rows'],
-                      'report_file': str(Path(args.out)/'neurips-program-2026.json'),
-                      'database_written': False, 'pdfs_downloaded': False}, ensure_ascii=False))

tests/fixtures/neurips2026-receipts.json

@@ -1,15 +0,0 @@
-{
-  "receipts": [
-    {
-      "url": "https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json",
-      "final_url": "https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json",
-      "status": 200,
-      "content_type": "application/json",
-      "file": "program-json.response",
-      "bytes": 20447021,
-      "sha256": "879293a863057fa05370443b3b5ddee12a10eeb37c810abc6c1d53f71f545997",
-      "started_at": "2026-11-09T17:57:23.825509+00:00",
-      "finished_at": "2026-11-09T17:57:40.125580+00:00"
-    }
-  ]
-}

tests/test_aamas2026.py

@@ -0,0 +1,167 @@
+import gzip
+import hashlib
+import json
+from pathlib import Path
+import tempfile
+import sqlite3
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors import aamas, aamas2026
+from paper_atlas.ingestion.records import ingest_file
+
+
+FIXTURE = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/aamas2026-fixture.json.gz').read_bytes()))
+EVIDENCE = {'url': aamas2026.URL, 'observed_at': FIXTURE['receipts']['receipts'][0]['finished_at']}
+
+
+class Aamas2026Tests(unittest.TestCase):
+    def parse(self, html=None):
+        return aamas2026.parse_contents(FIXTURE['contents'] if html is None else html, EVIDENCE)
+
+    def snapshots(self, folder):
+        folder.mkdir()
+        (folder/'contents.html').write_text(FIXTURE['contents'])
+        (folder/'receipts.json').write_text(json.dumps(FIXTURE['receipts']))
+        return folder
+
+    def test_every_publisher_title_author_affiliation_and_page_preserved(self):
+        rows, excluded, held = self.parse()
+        self.assertEqual((len(rows), len(excluded), len(held)), (639, 3, 1))
+        expected = {r['publisher_page']: r for r in FIXTURE['expected_entries']}
+        self.assertEqual(len(expected), 643)
+        all_rows = rows+excluded+held
+        self.assertEqual({r['metadata']['publisher_page'] for r in all_rows}, set(expected))
+        for r in all_rows:
+            e = expected[r['metadata']['publisher_page']]
+            self.assertEqual(r['title'], e['title'])
+            self.assertEqual(r['authors'], e['authors'])
+            self.assertEqual(r['metadata']['author_affiliations'], e['affiliations'])
+            self.assertEqual(r['metadata']['section_heading'], e['section'])
+        for r in rows:
+            self.assertIsNone(r['abstract'])
+            self.assertEqual(r['identifiers'], {})
+            if r['external_id'] != '2026/p3178':
+                self.assertEqual(r['status'], 'accepted')
+                self.assertNotIn('decision_date', r)
+
+    def test_explicit_withdrawal_preserves_status_date_and_no_pdf(self):
+        rows, _, _ = self.parse()
+        row, = [r for r in rows if r['external_id'] == '2026/p3178']
+        self.assertEqual(row['status'], 'withdrawn')
+        self.assertEqual(row['metadata']['withdrawal_date'], '2026-06-08')
+        self.assertNotIn('decision_date', row)
+        self.assertEqual(row['raw_decision'], 'Withdrawn June 8, 2026')
+        self.assertNotIn('paper_url', row['metadata'])
+        self.assertEqual(row['metadata']['publication_state'], 'withdrawn')
+        self.assertEqual(sum(r['status'] == 'accepted' for r in rows), 638)
+
+    def test_withdrawal_ingestion_retains_date_precision_without_invented_acceptance(self):
+        rows, _, _ = self.parse()
+        row, = [r for r in rows if r['external_id'] == '2026/p3178']
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp); path = root/'withdrawn.jsonl'; db = root/'sample.sqlite'
+            path.write_text(json.dumps(row, ensure_ascii=False)+'\n')
+            path.with_suffix('.manifest.json').write_text(json.dumps({
+                'source': 'aamas_proceedings', 'list_url': aamas2026.URL,
+                'venue': 'AAMAS', 'year': 2026, 'track': row['track'],
+                'count': 1, 'status': 'partial', 'missing_abstract_count': 1,
+                'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest()}))
+            ingest_file(db, path)
+            with sqlite3.connect(db) as c:
+                self.assertEqual(c.execute('SELECT status,raw_value,decision_date FROM decisions').fetchall(),
+                                 [('withdrawn', 'Withdrawn June 8, 2026', None)])
+                metadata = json.loads(c.execute('SELECT metadata_json FROM source_records').fetchone()[0])
+                self.assertEqual(metadata['withdrawal_date'], '2026-06-08')
+                self.assertEqual(metadata['decision_date_precision'], 'day')
+                self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
+
+    def test_retraction_is_not_accepted_or_withdrawn_and_retains_date(self):
+        rows, _, held = self.parse()
+        self.assertNotIn('2026/p238', {r['external_id'] for r in rows})
+        h, = held
+        self.assertEqual(h['publication_state'], 'retracted')
+        self.assertEqual(h['retraction_date'], '2026-06-08')
+        self.assertEqual(h['raw_publication_state'], 'Retracted June 8, 2026')
+        self.assertNotIn('status', h)
+        self.assertNotIn('paper_url', h['metadata'])
+        self.assertEqual(h['metadata']['source_evidence'], EVIDENCE)
+
+    def test_tracks_formats_and_prior_decisions_are_not_conflated(self):
+        rows, _, _ = self.parse()
+        counts = {t: sum(r['track'] == t for r in rows) for t in {r['track'] for r in rows}}
+        self.assertEqual(counts, {'proceedings_unspecified': 522, 'aaai_transfer': 19,
+                                 'blue_sky': 17, 'doctoral': 34, 'demo': 44, 'jaamas': 3})
+        for row in rows:
+            self.assertEqual(row['tags'][0]['namespace'], 'aamas_proceedings:publication_format')
+            if row['track'] == 'aaai_transfer':
+                self.assertIn('none inferred', row['metadata']['prior_submission_note'])
+                self.assertNotIn('decision_events', row)
+            if row['track'] == 'jaamas':
+                self.assertEqual(row['metadata']['publication_format'], 'journal_presentation')
+                self.assertIn('not another independent', row['metadata']['track_scope_note'])
+
+    def test_refuses_changed_year_section_page_link_author_and_retraction(self):
+        html = FIXTURE['contents']
+        replacements = [('AAMAS 2026:', 'AAMAS 2027:'),
+                        ('name="R" id="R"></a><font color="#010101">Research Paper Track',
+                         'name="R" id="R"></a><font color="#010101">Workshop Track'),
+                        ('(Page 4)', '(Page 1)'),
+                        ('../pdfs/HQQZ1937.pdf', 'https://other.example/paper.pdf'),
+                        ('Michael Wooldridge <i>', '<i>'),
+                        ('Retracted June 8, 2026', 'Withdrawn June 8, 2026')]
+        for old, new in replacements:
+            self.assertIn(old, html)
+            with self.subTest(change=old), self.assertRaises(ValueError):
+                self.parse(html.replace(old, new, 1))
+
+    def test_frozen_collection_is_partial_with_full_held_evidence(self):
+        with tempfile.TemporaryDirectory() as tmp, patch.object(aamas2026, 'fetch', side_effect=AssertionError('No HTTP')):
+            root = Path(tmp)
+            report = aamas.collect(2026, root/'out', snapshots=self.snapshots(root/'source'))
+            self.assertEqual(report['records'], 639)
+            self.assertEqual(report['status_counts'], {'accepted': 638, 'withdrawn': 1})
+            self.assertEqual(len(report['files']), 6)
+            self.assertEqual(len(report['held_publications']), 1)
+            self.assertFalse(report['database_written'])
+            self.assertFalse(report['pdfs_downloaded'])
+            for m in report['files']:
+                self.assertEqual(m['status'], 'partial')
+                self.assertEqual(m['missing_abstract_count'], m['count'])
+                self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
+            self.assertTrue((root/'out/aamas-proceedings-2026.json').is_file())
+
+    def test_frozen_response_rejects_tamper_unsafe_path_status_and_symlink(self):
+        for mode in ('tamper', 'path', 'status', 'symlink'):
+            with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp); source = self.snapshots(root/'source')
+                r = json.loads((source/'receipts.json').read_text())
+                if mode == 'tamper':
+                    (source/'contents.html').write_text('bad')
+                elif mode == 'path':
+                    r['receipts'][0]['file'] = '../outside.html'
+                elif mode == 'status':
+                    r['receipts'][0]['status'] = 403
+                else:
+                    (root/'outside.html').write_text(FIXTURE['contents'])
+                    (source/'contents.html').unlink()
+                    (source/'contents.html').symlink_to(root/'outside.html')
+                (source/'receipts.json').write_text(json.dumps(r))
+                with self.assertRaises(ValueError):
+                    aamas.collect(2026, root/'out', snapshots=source)
+                self.assertFalse((root/'out/aamas-proceedings-2026.json').exists())
+
+    def test_live_mode_fetches_only_html_and_preserves_receipt(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp); cache = root/'cache'; cache.mkdir()
+            receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': aamas2026.URL}
+            (cache/hashlib.sha256(aamas2026.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt))
+            with patch.object(aamas2026, 'fetch', return_value=FIXTURE['contents'].encode()) as fetch:
+                report = aamas.collect(2026, root/'out', cache=cache)
+            fetch.assert_called_once_with(aamas2026.URL, cache, 8*1024**2, delay=1)
+            self.assertEqual(report['source_receipts'][0]['response_receipt'], receipt)
+
+    def test_frozen_fixture_is_original_hashed_response(self):
+        r, = FIXTURE['receipts']['receipts']
+        self.assertEqual(hashlib.sha256(FIXTURE['contents'].encode()).hexdigest(), r['sha256'])
+        self.assertEqual(r['bytes'], len(FIXTURE['contents'].encode()))

tests/test_aamas_proceedings.py

@@ -99,7 +99,7 @@ class AamasProceedingsTests(unittest.TestCase):
                 aamas.collect(2012, root/'out', snapshots=source)
             self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
             with self.assertRaises(ValueError):
-                aamas.collect(2026, root/'future', snapshots=source)
+                aamas.collect(2027, root/'future', snapshots=source)
             self.assertFalse((root/'future').exists())
 
     def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):

tests/test_neurips_program.py

@@ -1,174 +0,0 @@
-import copy
-from collections import Counter
-import gzip
-import hashlib
-import json
-from pathlib import Path
-import tempfile
-import unittest
-from unittest.mock import patch
-from urllib.parse import parse_qs, urlsplit
-
-from paper_atlas.collectors import neurips_program as program
-
-FIXTURES = Path(__file__).parent/'fixtures'
-RAW = gzip.decompress((FIXTURES/'neurips2026-program.json.gz').read_bytes())
-RECEIPTS = json.loads((FIXTURES/'neurips2026-receipts.json').read_text())
-EVIDENCE = {'url': program.URL, 'observed_at': RECEIPTS['receipts'][0]['finished_at']}
-
-
-class NeuripsProgramTests(unittest.TestCase):
-    def sample(self):
-        rows = json.loads(RAW)['results']
-        oral = next(r for r in rows if r['id'] == 154866)
-        poster = next(r for r in rows if r['paper_url'] == oral['paper_url'] and r['event_type'] == 'Poster')
-        return copy.deepcopy(poster), copy.deepcopy(oral)
-
-    def payload(self, rows):
-        return json.dumps({'count': len(rows), 'next': None, 'previous': None, 'results': rows})
-
-    def snapshots(self, folder):
-        folder.mkdir()
-        (folder/'program-json.response').write_bytes(RAW)
-        (folder/'receipts.json').write_text(json.dumps(RECEIPTS))
-        return folder
-
-    def test_full_snapshot_tracks_and_all_titles_authors_affiliations(self):
-        rows, held, duplicates, folded = program.parse_program(RAW, EVIDENCE)
-        self.assertEqual((len(rows), len(held), len(duplicates), folded), (9004, 139, 1, 132))
-        self.assertEqual(Counter(r['track'] for r in rows),
-                         {'main': 7896, 'evaluations_datasets': 971, 'position': 137})
-        expected = {}
-        for e in json.loads(RAW)['results']:
-            if e['sourceurl'] in program.GROUPS and e['event_type'] == 'Poster':
-                forum = parse_qs(urlsplit(e['paper_url']).query)['id'][0]
-                expected[forum] = e
-        self.assertEqual(len(expected), 9004)
-        for r in rows:
-            e = expected[r['identifiers']['openreview']]
-            self.assertEqual(r['title'], e['name'])
-            self.assertEqual(r['authors'], [a['fullname'] for a in e['authors']])
-            self.assertEqual(r['metadata']['author_affiliations'], [a.get('institution') for a in e['authors']])
-            self.assertEqual(r['raw_decision'], e['decision'])
-            self.assertEqual(r['status'], 'accepted')
-            self.assertNotIn('decision_date', r)
-            self.assertIsNone(r['abstract'])
-            self.assertEqual(r['year'], 2026)
-        self.assertEqual(duplicates[0]['event_id'], 154866)
-
-    def test_same_forum_folds_presentations_and_preserves_signed_source_ids(self):
-        poster, oral = self.sample()
-        rows, held, duplicates, folded = program.parse_program(self.payload([oral, poster, oral]), EVIDENCE)
-        row, = rows
-        self.assertEqual((len(held), len(duplicates), folded), (0, 1, 1))
-        self.assertEqual({p['sourceid'] for p in row['metadata']['presentations']},
-                         {poster['sourceid'], oral['sourceid']})
-        self.assertLess(oral['sourceid'], 0)
-        self.assertEqual({t['value'] for t in row['tags']}, {'Poster', 'Oral'})
-        self.assertTrue(all(t['namespace'] == 'neurips_program:presentation_format' for t in row['tags']))
-
-    def test_identical_title_and_authors_in_distinct_forums_remain_distinct(self):
-        poster, _ = self.sample(); other = copy.deepcopy(poster)
-        other.update(id=999999, uid='0'*32, sourceid=999999, paper_url='https://openreview.net/forum?id=otherForum7',
-                     virtualsite_url='/virtual/2026/poster/999999')
-        rows, _, _, _ = program.parse_program(self.payload([poster, other]), EVIDENCE)
-        self.assertEqual(len(rows), 2)
-        self.assertEqual(len({r['external_id'] for r in rows}), 2)
-
-    def test_conflicting_event_ids_or_same_forum_bibliography_are_rejected(self):
-        poster, oral = self.sample()
-        for field in ('name', 'authors', 'sourceurl', 'decision'):
-            changed = copy.deepcopy(oral)
-            if field == 'authors':
-                changed['authors'][0]['fullname'] = 'Different Author'
-            elif field == 'sourceurl':
-                changed[field] = 'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track'
-            elif field == 'decision':
-                changed[field] = 'Accept (poster)'
-            else:
-                changed[field] += ' changed'
-            with self.subTest(field=field), self.assertRaises(ValueError):
-                program.parse_program(self.payload([poster, changed]), EVIDENCE)
-        changed = copy.deepcopy(poster); changed['name'] += ' changed'
-        with self.assertRaises(ValueError):
-            program.parse_program(self.payload([poster, changed]), EVIDENCE)
-
-    def test_unreviewed_origin_decision_visibility_and_link_are_held(self):
-        poster, _ = self.sample()
-        mutations = [{'sourceurl': 'journal-track/NeurIPS_2026-Posters_TMLR.xlsx'},
-                     {'sourceurl': 'neurips2026-bridging-the-future'},
-                     {'sourceurl': None}, {'sourceurl': 'https://openreview.net/group?id=NeurIPS.cc/2027/Conference'},
-                     {'decision': 'Reject'}, {'visible': False},
-                     {'paper_url': 'https://other.example/forum?id=anotherID7'}]
-        for mutation in mutations:
-            changed = copy.deepcopy(poster); changed.update(id=999999, virtualsite_url='/virtual/2026/poster/999999')
-            changed.update(mutation)
-            rows, held, _, _ = program.parse_program(self.payload([poster, changed]), EVIDENCE)
-            self.assertEqual((len(rows), len(held)), (1, 1))
-            self.assertEqual(held[0]['raw_event'], changed)
-
-    def test_rejects_truncation_pagination_invalid_url_and_missing_authors(self):
-        poster, _ = self.sample()
-        raw = json.loads(self.payload([poster]))
-        bad = [dict(raw, count=2), dict(raw, next='https://example.com/next'),
-               {k: v for k, v in raw.items() if k != 'previous'}]
-        for payload in bad:
-            with self.assertRaises(ValueError):
-                program.parse_program(json.dumps(payload), EVIDENCE)
-        for mutation in [{'authors': []}, {'virtualsite_url': '/virtual/2025/poster/154865'}, {'id': True}]:
-            changed = copy.deepcopy(poster); changed.update(mutation)
-            with self.assertRaises(ValueError):
-                program.parse_program(self.payload([changed]), EVIDENCE)
-
-    def test_frozen_collection_is_partial_and_no_http_or_database_write(self):
-        with tempfile.TemporaryDirectory() as tmp, patch.object(program, 'fetch', side_effect=AssertionError('No HTTP')):
-            root = Path(tmp); source = self.snapshots(root/'source')
-            report = program.collect(root/'out', snapshots=source)
-            self.assertEqual(report['records'], 9004)
-            self.assertEqual(len(report['files']), 3)
-            self.assertFalse(report['database_written'])
-            self.assertFalse(report['pdfs_downloaded'])
-            for m in report['files']:
-                self.assertEqual(m['status'], 'partial')
-                self.assertEqual(m['missing_abstract_count'], m['count'])
-                self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
-            with self.assertRaises(ValueError):
-                program.collect(root/'out', snapshots=source)
-
-    def test_frozen_receipt_tamper_path_symlink_status_and_duplicate(self):
-        for mode in ('tamper', 'path', 'symlink', 'status', 'duplicate'):
-            with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
-                root = Path(tmp); source = self.snapshots(root/'source')
-                receipts = copy.deepcopy(RECEIPTS)
-                if mode == 'tamper':
-                    (source/'program-json.response').write_text('bad')
-                elif mode == 'path':
-                    receipts['receipts'][0]['file'] = '../outside.json'
-                elif mode == 'symlink':
-                    (root/'outside.json').write_bytes(RAW)
-                    (source/'program-json.response').unlink()
-                    (source/'program-json.response').symlink_to(root/'outside.json')
-                elif mode == 'status':
-                    receipts['receipts'][0]['status'] = 403
-                else:
-                    receipts['receipts'].append(copy.deepcopy(receipts['receipts'][0]))
-                (source/'receipts.json').write_text(json.dumps(receipts))
-                with self.assertRaises(ValueError):
-                    program.collect(root/'out', snapshots=source)
-                self.assertFalse((root/'out/neurips-program-2026.json').exists())
-
-    def test_online_mode_only_requests_fixed_program_metadata(self):
-        poster, _ = self.sample(); raw = self.payload([poster]).encode()
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp); cache = root/'cache'; cache.mkdir()
-            receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': program.URL}
-            (cache/hashlib.sha256(program.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt))
-            with patch.object(program, 'fetch', return_value=raw) as fetch:
-                report = program.collect(root/'out', cache=cache)
-            fetch.assert_called_once_with(program.URL, cache, program.LIMIT, delay=1)
-            self.assertEqual(report['records'], 1)
-
-    def test_frozen_fixture_preserves_actual_response_bytes(self):
-        receipt, = RECEIPTS['receipts']
-        self.assertEqual(hashlib.sha256(RAW).hexdigest(), receipt['sha256'])
-        self.assertEqual(len(RAW), receipt['bytes'])