Keep publisher namespaces separate in RSS identity regression

Reviewed-on: #41

.gitea/workflows/check.yml

@@ -48,13 +48,26 @@ jobs:
               outside.mkdir()
               subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss-proceedings','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','event-identity','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','backfill','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
               smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
               subprocess.run([python,'-I','-B','-c',smoke],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','query','--db','installed.sqlite','--stats'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','annotate','init','--db','installed.sqlite'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','annotate','list','--db','installed.sqlite','--paper','1'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--db','installed.sqlite','--examples','0'],cwd=outside,check=True)
           PY

README.md

@@ -6,6 +6,10 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追
 
 项目仍在建设中。候选范围包含 43 个会议（含 ACM MM、MLSys），尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准，而不是仓库文件数量。
 
+历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配，保留主会/workshop及重复展示证据；运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
+
+RSS 2025 的 `collect rss-proceedings` 读取官方逐篇标题、完整作者、摘要和 DOI，并核对录用与演示节目单；不下载 PDF。15 个演示 ID 有明确来源，其余逐篇投稿类型保持未明确，不用标题或会话主题代替 track。运行、证据恢复和限制见 [RSS 元数据说明](docs/rss-proceedings.md)。
+
 ## 项目结构
 
 ```text
@@ -51,7 +55,8 @@ PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service fo
 PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --paper 123
 
 # 列出全部目标会议，包括尚未入库的项目
-paper-atlas coverage --db /data/paper-index/papers.sqlite \
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
+  --pg-service foundry-research --pg-schema research \
   --out /data/paper-index/collected/venue-coverage-current.json
 
 # 采集与导入分开；这里只采集到独立文件
@@ -59,6 +64,12 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
   --out /data/paper-index/collected
 ```
 
+早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别，保留原名与范围证据；代码支持不代表已补齐生产数据。
+
+EWSL、EPS、ECP、EWSP 的[具体前身卷与完整快照要求](docs/coverage/historical-predecessors-2026-11-09.md)单独核验，保留原始系列及事件/出版年份；旧候选文件缺少 ECP/EWSP 时会明确提示，避免静默遗漏。
+
+AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)目前核验了 2012 版式，可从缓存或已校验快照采集标题、作者及页面提供的摘要；发表形式单独保存，全文/扩展摘要中未明确的投稿 track 保持待明确，不根据会话主题推定主会轨道。
+
 `paper-atlas collect --help` 查看来源；`paper-atlas import --help` 查看导入方式。完整批次步骤见[使用指南](docs/usage.md)。
 
 共享实例名为 `foundry-postgres`，数据位于 `foundry_data` 库的 `research` schema；其他项目可按权限复用同一组表。库名通过 libpq service 配置、schema 通过 search_path 或 `--pg-schema` 指定，不写死在查询中。PostgreSQL 凭据放在服务器权限为 `0600` 的 libpq service 文件中；命令行只使用服务名称。`paper-atlas maintain postgres --help` 提供迁移入口：先完整核验 21 张业务表，再发布目标 schema。SQLite FTS5 内部表由 PostgreSQL GIN 索引替代。切换后 SQLite 为只读备份；`import records`、`import arxiv` 和 `maintain backfill` 使用 `--pg-service foundry-research` 写入主库，整个写入会话只允许一个协调写入者，会议文件失败时整体回滚并保留失败批次记录。
@@ -86,3 +97,12 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
 CI 在已授权的远程 runner 上运行测试、构建 wheel，并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤，不因代码测试通过而自动完成。
 
 开发通过工作分支和 PR 交付，由用户手动合并。
+
+早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。
+历史联合会议的既有届次归属可通过 [`maintain joint-editions`](docs/joint-editions.md) 先只读规划，再按计划哈希执行；保留论文与投稿决定来源，不重导入。
+同年多届的 `event_key`、显式迁移、旧文件刷新与查询边界见 [会议事件身份](docs/event-identity.md)。
+研究类型与自定义标签可通过 [`annotate`](docs/annotations.md) 显式初始化、添加和撤回；两者独立查询，保留证据及完整更正历史，不从会议轨道推断研究类型。
+
+整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
+
+已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md)：按完整证据生成只读计划，保留原投稿和决定 ID，事务性修正 edition、标识与覆盖，并防止旧整卷刷新覆盖分类。

docs/aamas-proceedings.md

@@ -0,0 +1,43 @@
+# AAMAS 官方论文集元数据
+
+`paper_atlas.collectors.aamas` 是按需运行的采集适配器，通过统一入口 `collect aamas-proceedings` 使用。目前只支持已独立核验的 **AAMAS 2012** HTML 版式；其他年份需先核对官方目录和逐篇结构，再增加适配，不能仅替换年份。
+
+## 来源与范围
+
+[官方论文集介绍](https://www.ifaamas.org/Proceedings/aamas2012/)及其 Full Papers、Extended Abstracts、Demos、Doctorial Consortium Abstracts 四个栏目分别列出 136、146、28、24 条，共 334 条。旧 DBLP 候选卷的 310 条不能替代这四类官方目录，也不能全部标记为主会全文。栏目数量来自 2026-11-09 冻结的原始页面；页面结构或数量发生变化时，采集器停止，留待重新核验。
+
+采集标题、逐篇作者、页面实际提供的摘要、官方条目 ID、目录锚点和 PDF 地址。**只保存 PDF 链接，不下载 PDF。** 全文页面的摘要藏在标题链接内部的 `abstract` 元素，作者链接里的 “has N papers” 是界面徽标，二者不会混入标题或姓名。未提供的摘要保存为 `null`，不会用标题、说明文字或其他论文填充。
+
+官方介绍说明了 general、robotics、virtual agents、innovative applications 等投稿轨道，但逐篇目录的主题会话不能独立证明某篇论文的投稿轨道。因此全文和扩展摘要均使用 `proceedings_unspecified`；演示、博士栏目使用 `demo`、`doctoral`。发表形式保留在 `metadata.publication_format` 和 `aamas_proceedings:publication_format` 标签中，会话原文单独保存。它们不是研究类型标注，也不会写入自定义 annotation。
+
+目录条目仅提供发表录用证据，不是原始决定信。`observed_at` 是页面采集时间，不能当作 2012 年的决定日期；官方介绍中的投稿、撤稿、录用汇总数不能识别逐篇拒稿或撤稿，采集器不会由总量相减生成这些决定。
+
+## 运行
+
+在已确认的远程运行环境中，使用 HTTP 缓存采集：
+
+```sh
+paper-atlas collect aamas-proceedings --year 2012 \
+  --cache /data/paper-index/cache/aamas \
+  --out /data/paper-index/collected/aamas-2012
+```
+
+也可使用已冻结的官方页面：
+
+```sh
+paper-atlas collect aamas-proceedings --year 2012 \
+  --snapshots /data/paper-index/collected/historical-next-source-evidence-v1 \
+  --out /data/paper-index/collected/aamas-2012
+```
+
+快照目录需含 `receipts.json`，其中每个响应包含原始 `url`、HTTP `status`、相对 `file`、`bytes`、`sha256` 和 `finished_at`。要求五个准确官方 URL 均唯一匹配且返回 200；验证全部字节数和哈希后才解析，不接受目录穿越或绝对文件路径。快照中其他来源的响应不会被当成 AAMAS 页面。
+
+输出为按数据库 track 分组的三个 JSONL 文件和对应 manifest、五份原始 HTML 证据，以及最后写出的 `aamas-proceedings-2012.json` 采集报告。锁文件避免同一输出目录并发运行。manifest 始终为 `partial`，记录发表形式数量和缺失摘要数；**采集命令不连接数据库，也不导入。**
+
+## 复核与恢复
+
+每条记录携带对应官方页面的 URL、采集时间、字节数、SHA-256 与输出内证据路径。重新运行前先检查已有采集报告和文件哈希；采集中断不等于主库已导入，报告缺失或哈希不一致的目录不能直接作为导入队列。输出中的原始证据应随文件一起保存。
+
+生产导入前仍需单独与主库核验稳定标识、标题、作者及已有投稿，按证据补充 DOI/arXiv 等标识；目录页上的作者计数徽标和会话 ID 不能作为这些标识。需要另建经过核验的增量清单、保护旧历史，完成独立验收后才能计入覆盖。跨来源同一论文不得仅因同题就新增或合并投稿。
+
+本功能的代码、测试或采集成功均不代表 AAMAS 2012 已导入生产，也不证明拒稿全集完整。全库最终去重、摘要和来源覆盖验收属于独立后续阶段。

docs/annotations.md

@@ -0,0 +1,107 @@
+# Research types and custom annotations
+
+Research type describes a claimed characteristic of the work (for example a survey,
+benchmark, empirical study or replication). It is independent of topic, paper length,
+conference track, presentation format and acceptance status. No type is inferred from
+those fields. Multiple source-specific or human classifications may coexist; no
+cross-platform vocabulary mapping or automatic classifier is provided.
+
+`annotation_events` stores both kinds with an explicit `kind` discriminator:
+`research_type` and `tag` are separate domains, even when namespace/value strings
+happen to coincide. Existing platform labels remain in `tags` / `paper_tags`.
+Each assertion carries namespace, value, actor, reason, evidence key, optional source
+record and evidence URL, and the time it was recorded. This is observation time, not
+paper publication time or a conference decision date.
+
+Values are not forced into a global taxonomy: `custom:method=empirical` and
+`platform:declared_type=empirical` retain separate provenance. An explicit platform
+assertion API requires a source record belonging to that paper and matching the
+namespace prefix. The manual CLI only accepts `custom:*`, and cannot retract platform
+assertions. No source type has been automatically assigned to existing papers.
+
+## Initialize explicitly
+
+On the published PostgreSQL dataset, using its protected service file:
+
+```sh
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas annotate init \
+  --pg-service foundry-research --pg-schema research
+```
+
+This creates one optional table, its indexes, and an upgrade marker in one transaction,
+using the existing dataset writer lock. It does not copy data, change existing papers
+or decisions, or create a PostgreSQL instance. Repeating it is safe; an unexplained
+pre-existing table/marker mismatch fails closed. Readers and ordinary imports do not
+install the extension. Legacy datasets remain readable. A SQLite **sample** uses
+`--db sample.sqlite`; the cutover SQLite backup remains protected against writes.
+
+## Add and retract evidence
+
+Use the canonical paper ID returned by `query`. For a reviewed classification:
+
+```sh
+paper-atlas annotate assert --pg-service foundry-research --pg-schema research \
+  --paper "$PAPER_ID" --kind research_type --namespace custom:method \
+  --value empirical --evidence-key manual:abstract-review \
+  --actor reviewer --reason 'Classified from the explicitly described study design' \
+  --request-id "$REQUEST_ID"
+```
+
+`$REQUEST_ID` must be a new, stable identifier for this action (for example a UUID).
+Retry with the same request ID and identical content after an interrupted response;
+it returns the original event ID. Reusing that ID with changed content is rejected.
+A topic annotation uses `--kind tag --namespace custom:topic --value robotics` instead.
+Optional `--source-record` and `--evidence-url` attach additional evidence; the source
+record must belong to this paper. URLs are retained as evidence and are not fetched.
+
+To withdraw one particular assertion, use its returned event ID:
+
+```sh
+paper-atlas annotate retract --pg-service foundry-research --pg-schema research \
+  --event "$EVENT_ID" --actor reviewer --reason 'Classification corrected after review' \
+  --request-id "$RETRACTION_REQUEST_ID"
+```
+
+Retraction adds an event referencing that assertion, preserving its original content.
+It does not remove another contributor's assertion of the same label, platform tags,
+source metadata or submission/decision history. A retracted assertion can be asserted
+again with a new request ID and evidence; it is a new observation. The API does not
+update or delete past annotation events. Database administrators still control direct
+SQL access. Work reviewed as the same paper may subsequently be merged by the existing
+identity workflow: annotation IDs, original paper IDs, evidence and retraction links
+are retained while the canonical paper reference follows the surviving entity.
+
+## Query and analyze
+
+```sh
+paper-atlas query --pg-service foundry-research --pg-schema research \
+  --research-type custom:method=empirical
+paper-atlas query --pg-service foundry-research --pg-schema research \
+  --tag custom:topic=robotics
+paper-atlas query --pg-service foundry-research --pg-schema research \
+  --paper "$PAPER_ID" --annotation-history
+paper-atlas annotate list --pg-service foundry-research --pg-schema research \
+  --paper "$PAPER_ID" --history
+```
+
+`--tag` includes original platform labels and active annotation tags. It never matches
+research types. `--research-type` matches active type assertions only. Paper details
+return separate `tags` (existing provenance), `annotation_tags`, and `research_types`;
+`--annotation-history` additionally includes retractions. A missing extension yields
+empty annotation results rather than changing the database.
+
+For direct SQL analysis, active research-type evidence is:
+
+```sql
+SELECT a.* FROM research.annotation_events a
+WHERE a.kind = 'research_type' AND a.retracts_id IS NULL
+  AND NOT EXISTS (SELECT 1 FROM research.annotation_events r WHERE r.retracts_id = a.id);
+```
+
+Several independent assertions can support one label; count distinct papers/labels
+when appropriate rather than counting evidence rows as papers. Namespaces stay
+separate. Standard paper queries decode the PostgreSQL text codec; direct SQL consumers
+must honor the dataset's documented codec, including NUL/SOH values. SQLite-to-PostgreSQL
+migration recognizes an installed annotation extension and verifies its full row hashes,
+constraints and identity sequence along with the original business tables. An old
+21-table snapshot stays a 21-table migration; no synthetic annotations are generated.

docs/coverage/README.md

@@ -188,3 +188,9 @@ MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明
 WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`：1995春季论文三目录均为38个相同条目ID，海报摘要53个条目与作者链接47个条目不齐；1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察，不是已导入或全部录用数量；主会、海报、教程等继续独立核验。
 
 WWW事件标识生产迁移已独立验收（报告SHA `f5dbb8a8`）：保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源（约223KB），只作为后续维护式采集器的输入；尚未补导早期WWW论文，也未启用可选注解表。
+
+IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijcai-sections-2026-11-08.md) 和 [稳定 ID 映射](ijcai-sections-2026-11-08.json)。已定位 345 条非 main 分类及 2022 官方 HTML 缺失但官方文件仍存在的一条记录；生产分类尚未修正，不能把整卷数量作为 main 覆盖率。
+
+2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度；它使用当前生产edition/别名快照，保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
+
+[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希，排除4个COLING卷对ACL缺口的误匹配，并区分会议举办记录、目录可获取性与论文级覆盖；尚未导入或关闭这些年度缺口。

docs/coverage/historical-gaps-2026-11-09.json

@@ -0,0 +1,867 @@
+{
+  "snapshot_at": "2026-11-08 16:34:40.741334+00:00",
+  "scope": "Official first-pass conference-year census vs current populated editions and real aliases. Missing year means no imported attempt in that year, not missing paper count or proof all other years complete. Includes announced/nonarchival exceptions below.",
+  "venues": [
+    {
+      "venue": "NeurIPS",
+      "expected_years": 40,
+      "observed_expected_years": 39,
+      "missing_years": [
+        2026
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "NeurIPS"
+      ]
+    },
+    {
+      "venue": "ICML",
+      "expected_years": 43,
+      "observed_expected_years": 14,
+      "missing_years": [
+        1980,
+        1983,
+        1985,
+        1987,
+        1988,
+        1989,
+        1990,
+        1991,
+        1992,
+        1993,
+        1994,
+        1995,
+        1996,
+        1997,
+        1998,
+        1999,
+        2000,
+        2001,
+        2002,
+        2003,
+        2004,
+        2005,
+        2006,
+        2007,
+        2008,
+        2009,
+        2010,
+        2011,
+        2012
+      ],
+      "missing_count": 29,
+      "canonical_matches": [
+        "ICML"
+      ]
+    },
+    {
+      "venue": "ICLR",
+      "expected_years": 14,
+      "observed_expected_years": 11,
+      "missing_years": [
+        2014,
+        2015,
+        2016
+      ],
+      "missing_count": 3,
+      "canonical_matches": [
+        "ICLR"
+      ]
+    },
+    {
+      "venue": "COLM",
+      "expected_years": 3,
+      "observed_expected_years": 3,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "COLM"
+      ]
+    },
+    {
+      "venue": "COLT",
+      "expected_years": 39,
+      "observed_expected_years": 39,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "COLT"
+      ]
+    },
+    {
+      "venue": "AISTATS",
+      "expected_years": 29,
+      "observed_expected_years": 25,
+      "missing_years": [
+        1985,
+        1989,
+        1991,
+        1993
+      ],
+      "missing_count": 4,
+      "canonical_matches": [
+        "AISTATS"
+      ]
+    },
+    {
+      "venue": "UAI",
+      "expected_years": 42,
+      "observed_expected_years": 42,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "UAI"
+      ]
+    },
+    {
+      "venue": "CLeaR",
+      "expected_years": 5,
+      "observed_expected_years": 5,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "CLEAR"
+      ]
+    },
+    {
+      "venue": "ACL",
+      "expected_years": 64,
+      "observed_expected_years": 48,
+      "missing_years": [
+        1963,
+        1964,
+        1965,
+        1966,
+        1967,
+        1968,
+        1969,
+        1970,
+        1971,
+        1972,
+        1973,
+        1974,
+        1975,
+        1976,
+        1977,
+        1978
+      ],
+      "missing_count": 16,
+      "canonical_matches": [
+        "ACL",
+        "ACL+COLING",
+        "ACL+EACL"
+      ]
+    },
+    {
+      "venue": "EMNLP",
+      "expected_years": 31,
+      "observed_expected_years": 31,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "CoNLL+EMNLP",
+        "EMNLP"
+      ]
+    },
+    {
+      "venue": "NAACL",
+      "expected_years": 18,
+      "observed_expected_years": 18,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "NAACL"
+      ]
+    },
+    {
+      "venue": "EACL",
+      "expected_years": 19,
+      "observed_expected_years": 19,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "ACL+EACL",
+        "EACL"
+      ]
+    },
+    {
+      "venue": "COLING",
+      "expected_years": 31,
+      "observed_expected_years": 28,
+      "missing_years": [
+        1971,
+        1976,
+        1978
+      ],
+      "missing_count": 3,
+      "canonical_matches": [
+        "ACL+COLING",
+        "COLING",
+        "COLING+LREC"
+      ]
+    },
+    {
+      "venue": "CoNLL",
+      "expected_years": 30,
+      "observed_expected_years": 30,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "CoNLL",
+        "CoNLL+EMNLP"
+      ]
+    },
+    {
+      "venue": "CVPR",
+      "expected_years": 39,
+      "observed_expected_years": 14,
+      "missing_years": [
+        1983,
+        1985,
+        1986,
+        1988,
+        1989,
+        1991,
+        1992,
+        1993,
+        1994,
+        1996,
+        1997,
+        1998,
+        1999,
+        2000,
+        2001,
+        2003,
+        2004,
+        2005,
+        2006,
+        2007,
+        2008,
+        2009,
+        2010,
+        2011,
+        2012
+      ],
+      "missing_count": 25,
+      "canonical_matches": [
+        "CVPR"
+      ]
+    },
+    {
+      "venue": "ICCV",
+      "expected_years": 20,
+      "observed_expected_years": 7,
+      "missing_years": [
+        1987,
+        1988,
+        1990,
+        1993,
+        1995,
+        1998,
+        1999,
+        2001,
+        2003,
+        2005,
+        2007,
+        2009,
+        2011
+      ],
+      "missing_count": 13,
+      "canonical_matches": [
+        "ICCV"
+      ]
+    },
+    {
+      "venue": "ECCV",
+      "expected_years": 19,
+      "observed_expected_years": 4,
+      "missing_years": [
+        1990,
+        1992,
+        1994,
+        1996,
+        1998,
+        2000,
+        2002,
+        2004,
+        2006,
+        2008,
+        2010,
+        2012,
+        2014,
+        2016,
+        2026
+      ],
+      "missing_count": 15,
+      "canonical_matches": [
+        "ECCV"
+      ]
+    },
+    {
+      "venue": "AAAI",
+      "expected_years": 40,
+      "observed_expected_years": 40,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "AAAI",
+        "AAAI+IAAI"
+      ]
+    },
+    {
+      "venue": "IJCAI",
+      "expected_years": 35,
+      "observed_expected_years": 35,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "ECAI+IJCAI",
+        "IJCAI"
+      ]
+    },
+    {
+      "venue": "ECAI",
+      "expected_years": 29,
+      "observed_expected_years": 27,
+      "missing_years": [
+        1974,
+        1980
+      ],
+      "missing_count": 2,
+      "canonical_matches": [
+        "ECAI",
+        "ECAI+IJCAI"
+      ]
+    },
+    {
+      "venue": "KDD",
+      "expected_years": 32,
+      "observed_expected_years": 32,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "KDD"
+      ]
+    },
+    {
+      "venue": "SIGIR",
+      "expected_years": 50,
+      "observed_expected_years": 50,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "SIGIR"
+      ]
+    },
+    {
+      "venue": "WWW",
+      "expected_years": 33,
+      "observed_expected_years": 27,
+      "missing_years": [
+        1994,
+        1996,
+        1997,
+        1998,
+        1999,
+        2000
+      ],
+      "missing_count": 6,
+      "canonical_matches": [
+        "WWW"
+      ]
+    },
+    {
+      "venue": "WSDM",
+      "expected_years": 19,
+      "observed_expected_years": 19,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "WSDM"
+      ]
+    },
+    {
+      "venue": "RecSys",
+      "expected_years": 20,
+      "observed_expected_years": 19,
+      "missing_years": [
+        2026
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "RecSys"
+      ]
+    },
+    {
+      "venue": "CIKM",
+      "expected_years": 35,
+      "observed_expected_years": 34,
+      "missing_years": [
+        2026
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "CIKM"
+      ]
+    },
+    {
+      "venue": "ICDM",
+      "expected_years": 26,
+      "observed_expected_years": 25,
+      "missing_years": [
+        2026
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "ICDM"
+      ]
+    },
+    {
+      "venue": "SDM",
+      "expected_years": 26,
+      "observed_expected_years": 25,
+      "missing_years": [
+        2026
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "SDM"
+      ]
+    },
+    {
+      "venue": "ECML-PKDD",
+      "expected_years": 38,
+      "observed_expected_years": 33,
+      "missing_years": [
+        1986,
+        1987,
+        1988,
+        1989,
+        1991
+      ],
+      "missing_count": 5,
+      "canonical_matches": [
+        "ECML",
+        "ECML-PKDD",
+        "PKDD"
+      ]
+    },
+    {
+      "venue": "ICRA",
+      "expected_years": 43,
+      "observed_expected_years": 43,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "ICRA"
+      ]
+    },
+    {
+      "venue": "IROS",
+      "expected_years": 39,
+      "observed_expected_years": 35,
+      "missing_years": [
+        1988,
+        1989,
+        1990,
+        2026
+      ],
+      "missing_count": 4,
+      "canonical_matches": [
+        "IROS"
+      ]
+    },
+    {
+      "venue": "RSS",
+      "expected_years": 22,
+      "observed_expected_years": 20,
+      "missing_years": [
+        2025,
+        2026
+      ],
+      "missing_count": 2,
+      "canonical_matches": [
+        "RSS"
+      ]
+    },
+    {
+      "venue": "CoRL",
+      "expected_years": 10,
+      "observed_expected_years": 9,
+      "missing_years": [
+        2026
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "CORL"
+      ]
+    },
+    {
+      "venue": "ICAPS",
+      "expected_years": 37,
+      "observed_expected_years": 28,
+      "missing_years": [
+        1990,
+        1991,
+        1992,
+        1993,
+        1995,
+        1997,
+        1999,
+        2001,
+        2026
+      ],
+      "missing_count": 9,
+      "canonical_matches": [
+        "AIPS",
+        "ICAPS"
+      ]
+    },
+    {
+      "venue": "KR",
+      "expected_years": 23,
+      "observed_expected_years": 23,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "KR"
+      ]
+    },
+    {
+      "venue": "AAMAS",
+      "expected_years": 25,
+      "observed_expected_years": 22,
+      "missing_years": [
+        2012,
+        2025,
+        2026
+      ],
+      "missing_count": 3,
+      "canonical_matches": [
+        "AAMAS"
+      ]
+    },
+    {
+      "venue": "ICASSP",
+      "expected_years": 51,
+      "observed_expected_years": 49,
+      "missing_years": [
+        1977,
+        2026
+      ],
+      "missing_count": 2,
+      "canonical_matches": [
+        "ICASSP"
+      ]
+    },
+    {
+      "venue": "INTERSPEECH",
+      "expected_years": 39,
+      "observed_expected_years": 39,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "ECST",
+        "EUROSPEECH",
+        "ICSLP",
+        "INTERSPEECH"
+      ]
+    },
+    {
+      "venue": "ISMIR",
+      "expected_years": 27,
+      "observed_expected_years": 26,
+      "missing_years": [
+        2026
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "ISMIR"
+      ]
+    },
+    {
+      "venue": "MICCAI",
+      "expected_years": 29,
+      "observed_expected_years": 29,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "MICCAI"
+      ]
+    },
+    {
+      "venue": "FAccT",
+      "expected_years": 9,
+      "observed_expected_years": 9,
+      "missing_years": [],
+      "missing_count": 0,
+      "canonical_matches": [
+        "FAccT"
+      ]
+    },
+    {
+      "venue": "MLSys",
+      "expected_years": 9,
+      "observed_expected_years": 8,
+      "missing_years": [
+        2018
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "MLSys"
+      ]
+    },
+    {
+      "venue": "MM",
+      "expected_years": 34,
+      "observed_expected_years": 33,
+      "missing_years": [
+        2026
+      ],
+      "missing_count": 1,
+      "canonical_matches": [
+        "MM"
+      ]
+    }
+  ],
+  "expected_venue_years": 1266,
+  "missing_venue_years": 150,
+  "venues_with_missing_years": 25,
+  "known_nonpublication_exceptions": [
+    {
+      "venue": "NeurIPS",
+      "year": 2026,
+      "reason": "announced;not in proceedings index at census"
+    },
+    {
+      "venue": "CoRL",
+      "year": 2026,
+      "reason": "not in publisher proceedings index at census"
+    },
+    {
+      "venue": "MLSys",
+      "year": 2018,
+      "reason": "first non-archival SysML meeting; not a missing formal publication volume"
+    }
+  ],
+  "historical_missing_years_excluding_nonarchival_2018": 135,
+  "historical_venues_with_missing_years": 16,
+  "historical_gaps": [
+    {
+      "venue": "ICML",
+      "years": [
+        1980,
+        1983,
+        1985,
+        1987,
+        1988,
+        1989,
+        1990,
+        1991,
+        1992,
+        1993,
+        1994,
+        1995,
+        1996,
+        1997,
+        1998,
+        1999,
+        2000,
+        2001,
+        2002,
+        2003,
+        2004,
+        2005,
+        2006,
+        2007,
+        2008,
+        2009,
+        2010,
+        2011,
+        2012
+      ]
+    },
+    {
+      "venue": "ICLR",
+      "years": [
+        2014,
+        2015,
+        2016
+      ]
+    },
+    {
+      "venue": "AISTATS",
+      "years": [
+        1985,
+        1989,
+        1991,
+        1993
+      ]
+    },
+    {
+      "venue": "ACL",
+      "years": [
+        1963,
+        1964,
+        1965,
+        1966,
+        1967,
+        1968,
+        1969,
+        1970,
+        1971,
+        1972,
+        1973,
+        1974,
+        1975,
+        1976,
+        1977,
+        1978
+      ]
+    },
+    {
+      "venue": "COLING",
+      "years": [
+        1971,
+        1976,
+        1978
+      ]
+    },
+    {
+      "venue": "CVPR",
+      "years": [
+        1983,
+        1985,
+        1986,
+        1988,
+        1989,
+        1991,
+        1992,
+        1993,
+        1994,
+        1996,
+        1997,
+        1998,
+        1999,
+        2000,
+        2001,
+        2003,
+        2004,
+        2005,
+        2006,
+        2007,
+        2008,
+        2009,
+        2010,
+        2011,
+        2012
+      ]
+    },
+    {
+      "venue": "ICCV",
+      "years": [
+        1987,
+        1988,
+        1990,
+        1993,
+        1995,
+        1998,
+        1999,
+        2001,
+        2003,
+        2005,
+        2007,
+        2009,
+        2011
+      ]
+    },
+    {
+      "venue": "ECCV",
+      "years": [
+        1990,
+        1992,
+        1994,
+        1996,
+        1998,
+        2000,
+        2002,
+        2004,
+        2006,
+        2008,
+        2010,
+        2012,
+        2014,
+        2016
+      ]
+    },
+    {
+      "venue": "ECAI",
+      "years": [
+        1974,
+        1980
+      ]
+    },
+    {
+      "venue": "WWW",
+      "years": [
+        1994,
+        1996,
+        1997,
+        1998,
+        1999,
+        2000
+      ]
+    },
+    {
+      "venue": "ECML-PKDD",
+      "years": [
+        1986,
+        1987,
+        1988,
+        1989,
+        1991
+      ]
+    },
+    {
+      "venue": "IROS",
+      "years": [
+        1988,
+        1989,
+        1990
+      ]
+    },
+    {
+      "venue": "RSS",
+      "years": [
+        2025
+      ]
+    },
+    {
+      "venue": "ICAPS",
+      "years": [
+        1990,
+        1991,
+        1992,
+        1993,
+        1995,
+        1997,
+        1999,
+        2001
+      ]
+    },
+    {
+      "venue": "AAMAS",
+      "years": [
+        2012,
+        2025
+      ]
+    },
+    {
+      "venue": "ICASSP",
+      "years": [
+        1977
+      ]
+    }
+  ],
+  "current_2026_gaps": 14
+}

docs/coverage/historical-gaps-2026-11-09.md

@@ -0,0 +1,58 @@
+# 历史年度缺口的当前核对
+
+生产只读快照：2026-11-08 16:34:40.741334 UTC（北京时间11月9日00:34），foundry_data/research。对比已核实的43份首轮官方会议年度清单，使用当前有投稿记录的edition及真实venue_aliases，不沿用旧固定快照的missing_years。
+
+- 清单共1266个会议—年份位置，当前有150个位置尚无投稿记录，涉及25个目标会议。
+- 其中2025年及以前136个；排除已明确非存档的SysML2018首届，剩**135个历史年度待补采或核实，涉及16个会议**。
+- 另有14个2026年度待核实。已知NeurIPS2026和CoRL2026在首轮核查时尚未进入论文集索引，不能一律当已公开论文遗漏；其他年度也需逐个核实发布时间。
+
+这些是“整年没有任何投稿记录”的数量，不是缺论文篇数，也不是完整性百分比。一个年份只有少数记录、只有公开拒稿或缺了某个track，仍会被计作“已有年度”；这些更细缺口及公开拒稿的逐年可获得性尚须继续核验。会议别名共享的联合届次不能当成两批独立论文，未见录用也不能推断拒稿。
+
+| 会议 | 历史缺口年份数 |
+| --- | ---: |
+| ICML | 29 |
+| CVPR | 25 |
+| ACL | 16 |
+| ECCV | 14 |
+| ICCV | 13 |
+| ICAPS | 8 |
+| WWW | 6 |
+| ECML-PKDD | 5 |
+| AISTATS | 4 |
+| ICLR、COLING、IROS | 各3 |
+| ECAI、AAMAS | 各2 |
+| RSS、ICASSP | 各1 |
+
+[完整逐年结果](historical-gaps-2026-11-09.json) 保留43目标的官方预期、现有年份数、缺失年份与规范会议匹配。原始只读生产导出保存在 `/data/paper-index/collected/historical-gap-current-20261109-v1.json`。 导出SHA256 `580cdea6dd9869adafe9bc28fb75cf3854bb290ad65aa5a600d9bc368a528b93`。年度对照清单为 `docs/coverage/inventory-2026-11-08.json`；候选来源文件绑定的输入哈希与本目录缺口JSON逐字节一致。来源清单的年份和哈希沿用相应 `*-editions-2026-11-08.json`，不是本轮重新访问所有会议网站；2026发布状态需要后续更新。
+
+## 候选来源与下一步
+
+另一次只读目录核查在现有、已校验的DBLP快照中，为135个历史年度中的**85个**匹配到初步候选论文集，另**50个**尚未按本轮规则匹配到，需要继续识别目录键或寻找官方来源。未匹配不等于会议没举办、未出版或没有公开来源。
+
+[候选卷清单](historical-source-candidates-2026-11-09.json) 保留卷key、标题、出版年份、原始bibliographic counts和出版者链接。初筛使用目标family映射内的四位卷key年份或卷标题唯一显式年份，再按具体会议身份复核；两位键、出版年份与会议年份差异、前身系列及其他目录命名可能漏匹配。原始计数可能含workshop、前言、索引或多个卷，**不能当可直接导入的录用论文数量**。
+
+后续[来源范围复核](historical-source-scope-2026-11-09.md)排除了4个误归ACL的COLING卷，因此候选年度由89修正为85、未匹配年度由46增至50。原卷及排除理由保留在JSON中；135个历史年度缺口不因此改变。ACL1963–1978与COLING1971/1976/1978在当前Anthology目录未列出，继续记为来源待核实，不能一律称作已公开但漏采的论文集。
+
+优先核查已有候选卷较集中的CVPR22年、ICCV12年、ECCV14年和ICML25年。其中ICML早期workshop名称可能属于官方主系列前身，也可能是附属workshop，须对照官方届次证据逐卷区分，不能靠标题包含workshop一刀切。所有候选须经过范围、稳定标识、重复/冲突、原始字段和覆盖预检；这一步没有导入候选卷。
+
+## 复核方法
+
+只读获取当前有记录的edition，不连接papers大文本或标签表：
+
+```sql
+WITH counts AS (
+  SELECT edition_id,count(*) AS submissions
+  FROM submissions GROUP BY edition_id
+)
+SELECT e.id,e.venue_id,v.acronym,e.year,e.track,e.event_key,counts.submissions
+FROM editions e JOIN venues v ON v.id=e.venue_id
+JOIN counts ON counts.edition_id=e.id ORDER BY e.id;
+
+SELECT venue_id,alias FROM venue_aliases ORDER BY venue_id,alias;
+```
+
+逐目标取规范会议名或别名相等的edition年份集合，与该目标edition_census.expected_years做集合差；再把2026与明确非存档的SysML2018单列。保留原始导出与校验哈希，不把每个matched year误当逐篇/逐track完整。
+
+与本清单同一阶段的WWW补采只补1995年现有poster轨道，不新增缺失年度，所以即使其15条增量已验收，135这个年度计数也不会随之下降。早期1994年的50条正式展示候选、307条混合节目候选共357条仍需核实；其中并非每条都是论文。
+
+本清单用于明确第14步剩余范围，不代表第14步完成，也不替代所有导入结束后的第16步全库去重、摘要、约束和实际查询验收。

docs/coverage/historical-predecessors-2026-11-09.md

@@ -0,0 +1,53 @@
+# EWSL 与规划会议前身的来源范围
+
+本项扩展 DBLP 元数据适配：核对具体历史卷，保留 EWSL、EPS、ECP、EWSP 原始系列名称，并使用已有会议别名关系关联 ECML-PKDD 或 ICAPS。它不把历史论文的原始会议名称替换成后来的会议，也不从卷的存在推断有逐篇数据。
+
+## 证据与已核验边界
+
+[ECML-PKDD 2011 官方历史页](https://ecmlpkdd2011.org/pastConferences.php)与 [ICAPS 官方系列历史](https://www.icaps-conference.org/conference-series/)分别提供系列前身关系。2026-11-09 冻结页面 SHA-256 为：
+
+- ECML-PKDD：`c3ecdb032aeacf6a365aace5457773827d1405872ad9bc6be6fc05851d5e68d6`。
+- ICAPS：`de1d217660cb819825f58112cdaf774d0f68da6ed87e3e90a59055e52f152081`。
+
+与固定 DBLP 2026-11-01 快照的完整 proceedings 目录对照后，仅对下列具体卷识别前身主系列：
+
+| 原始系列 | 事件年 | DBLP 卷 key | 卷出版年 | 原始逐篇书目计数 |
+|---|---:|---|---:|---:|
+| EWSL | 1987 | `conf/ecml/1987` | 1987 | 16 |
+| EWSL | 1988 | `conf/ecml/1988` | 1988 | 21 |
+| EWSL | 1989 | `conf/ecml/1989` | 1989 | 0 |
+| EWSL | 1991 | `conf/ecml/1991` | 1991 | 44 |
+| EPS | 1990 | `conf/aips/1990` | 1990 | 48 |
+| ECP | 1997 | `conf/ecp/1997` | 1997 | 35 |
+| ECP | 1999 | `conf/ecp/1999` | 2000 | 28 |
+| EWSP | 1991 | `conf/ewsp/1991` | 1991 | 10 |
+
+合计 202 条是原始 `inproceedings` 书目计数，尚未经过前言等材料排除、身份核查与生产验收，不能作为新论文数量。EWSL 1989 在当前 catalogue 中有卷信息，但未匹配到 `inproceedings` 条目；其他记录类型和其他来源仍待核查。代码支持它的范围规则不表示已补上这一年的论文。
+
+核验同时要求来源 family、booktitle 基名、空轨道后缀、完整 proceedings key、完整标题及卷出版年与固定证据一致。邻近年份、标题附加内容、`w` 卷后缀或工作坊 booktitle 均不能继承此范围。ECP、EWSP 及 EPS 的未知卷保持待审。原有 AIPS、ICML/IROS、AISTATS/COLT 规则保持各自边界。
+
+EWSL 的主轨道仍属于 `EWSL`，通过 `ECML-PKDD` 别名纳入家族查询；EPS、ECP、EWSP 保留原名，通过 `ICAPS` 别名关联。`main` 在此表示该原始系列的主轨道。原始 booktitle、作者、论文书目年、卷标题、卷出版年、事件年及历史页 URL 分开保存在 metadata 中。例如 ECP 1999 的事件年 1999 与卷出版年 2000 都保留，不用其中一个覆盖另一个。
+
+## 候选文件与完整快照
+
+旧的窄范围 `dblp-ai-candidates.jsonl` 未纳入 `ecp`、`ewsp` family，但完整 catalogue 的 proceedings 与 volume_counts 中有它们。新的 catalogue 导出会纳入这两个 family。
+
+选择 ICAPS 且截止年达到 EWSP 1991 或 ECP 1997 时，若候选 manifest 缺少相应 family，`collect dblp --candidates` 会明确报错，要求使用原始完整 `--archive`。错误发生在写输出之前，不会把遗漏当作成功采集。选择其他会议或截止 EPS 1990 的旧候选采集不受这项新家族检查影响。
+
+在已确认的远程运行环境中使用完整快照：
+
+```sh
+paper-atlas collect dblp \
+  --catalogue /data/paper-index/collected/dblp/dblp-catalogue.sqlite \
+  --archive /path/to/pinned-dblp.xml.gz --dtd /path/to/pinned-dblp.dtd \
+  --venue ECML-PKDD --venue ICAPS --through-year 2001 \
+  --out /data/paper-index/collected/historical-predecessors
+```
+
+完整快照与 DTD 均沿用现有固定身份校验。该命令也会输出已覆盖的其他年度；生产导入前须单独筛选目标卷与年度，检查 raw source、前言材料、稳定标识、标题、作者及现有投稿，不能直接导入整个输出目录。不会下载论文 PDF。
+
+## 验证和当前限制
+
+回归检查使用独立保存的八卷原始目录 fixture，合成论文只用于隔离测试；不代表真实逐篇来源。检查原始系列/别名、完整标题和年份边界、旧候选文件显式失败、扩展 family 的 catalogue 导出、原生导入后的元数据与关系。
+
+代码、测试、采集和生产导入是不同阶段。新范围不表示本批数据已采集或已导入，更不表示公开拒稿、摘要或最终去重完整。原始书目仅提供发表条目证据，不是原始决定信；没有记录不能推断为拒稿或未举办。后续各批需保留独立验收报告，保护既有跨会投稿、决定与标签历史。

docs/coverage/historical-primary-series-2026-11-09.md

@@ -0,0 +1,33 @@
+# ICML 与 IROS 早期主系列的来源范围
+
+本轮修正采集阶段的届次识别，不修改任何现有生产投稿轨道或历史。正式主系列曾以Workshop命名，不等于附属workshop；也不能反过来把所有旧workshop升为主会。
+
+## 已核实的精确范围
+
+| 会议 | 论文集键 | 依据与行为 |
+| --- | --- | --- |
+| ICML | `conf/icml/1988`、`1989`、`1990`、`1991`、`1992` | [官方历史](https://icml.cc/Conferences/2008/past_icmls.shtml.html)将其列为第5–9届，年份与举办地对应固定目录中的完整标题。1989/1991/1992的Workshop名称不再导致主系列被默认排除；1988/1990保留main并增加来源范围证据 |
+| IROS | `conf/iros/1988`、`1989`、`1990` | [IEEE RAS官方历届目录](https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/)列出三届与论文集链接，地点及日期对应固定目录标题；这些是IROS主系列 |
+
+规则同时要求family、booktitle基础别名（ML或IROS）、空轨道后缀、完整精确论文集键、匹配的事件年份及固定完整标题。仍执行原有书目年份核查：与事件年不同且标题无明确事件年时拒绝推断。ML91原始标题没有四位年份，使用精确1991卷键、完整标题、逐篇书目年和官方第8届记录，不从两位标题代码猜世纪。
+
+`1991w`/`1988w`等卷键、booktitle中的Workshops后缀、不同届次/地点/完整标题、未列入的年份均不会获得上述主系列豁免。AISTATS/COLT既有精确规则保留；真正的附属workshop仍受默认不采集workshop的边界约束。
+
+## 原始字段、决定与后续导入
+
+采集器仍保存原始proceedings标题（包括Workshop）、书目年、原始作者名、完整来源书目字段及稳定标识；新增 `track_scope_note` 与官方 `track_scope_evidence_url`。接受状态仍只源自该论文的inproceedings书目证据；不从历史目录生成占位论文或拒稿，也不重新导入已经存在的记录。
+
+新规则的输出必须重新做原始来源/标题/作者/稳定标识与现有库冲突预检，再按增量导入。此前被隔离的ICML1988/1990记录不能仅凭代码规则而直接解除全部冲突。当前正在运行的历史批次使用原固定版本和文件，不能在运行中替换其源码或输入。
+
+## 验证与证据
+
+新增回归覆盖：ICML三届Workshop名称、IROS最初三届、ICML原Conference名称的两届、错误卷键/后缀/标题/未知年份，以及归一化记录经过原生文件导入后保留轨道证据。构建与测试仅在已授权的ces-control远程runner执行，本机仅做语法和文本/JSON一致性检查；结果以相应commit的CI回执为准。
+
+官方HTML在工作证据目录 `paper-index/historical-primary-series-evidence-v1/`，附响应状态、字节数、时间与SHA256；仅取元数据网页，没有下载论文全文。
+
+| 来源 | UTC采样时间 | HTTP | 字节数 | SHA256 |
+| --- | --- | ---: | ---: | --- |
+| [icml-history](https://icml.cc/Conferences/2008/past_icmls.shtml.html) | 2026-11-08T18:36:43.043135+00:00 | 200 | 11993 | `80cce886fe985551339a52b13c4077acf827e8fcc01555f854dc24077bdebdf0` |
+| [iros-history](https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/) | 2026-11-08T18:36:45.687171+00:00 | 200 | 466059 | `fdb4594621e68567e1099f5c88aae53de835b7b8146025148c3b1968df990697` |
+
+本次不关闭历史年度缺口，也不替代全部导入后的全库去重、来源/年度/track覆盖、摘要和实际查询验收。

docs/coverage/historical-source-scope-2026-11-09.md

@@ -0,0 +1,32 @@
+# 早期 NLP 来源范围复核
+
+核查时间：2026-11-08 18:27 UTC（北京时间11月9日）。本轮只修正来源候选和记录可获得性，没有操作生产数据库。
+
+## 跨会议候选修正
+
+先前候选清单将 `conf/coling/1965`、`1967`、`1969`、`1973` 匹配到 ACL 年会缺口。这四卷标题明确属于 COLING；[COLING 官方 Anthology 目录](https://aclanthology.org/venues/coling/)将它们列在 COLING 下。出版者写作 ACL 或目标映射包含 COLING，不足以证明某一年是 ACL 联办届次。
+
+从 ACL 候选中排除上述四卷，原始卷字段和排除理由保留在[候选JSON](historical-source-candidates-2026-11-09.json)的 `excluded_cross_venue_candidates` 中。135个历史会议—年份位置仍未关闭；有初步候选来源的年度由89修正为85，尚未匹配的由46增至50。这些数字是目录初筛结果，不是已验证可导入的论文数。
+
+联合届次仍按具体年度的明确会议身份处理，例如有官方联办证据的卷可以属于多个会议。不能一律排除跨family卷，也不能把联合卷重复导入为两批论文。
+
+## 来源待核实的早期年份
+
+| 范围 | 本轮官方来源观察 | 处理 |
+| --- | --- | --- |
+| ACL 1963–1978，共16年 | [ACL 官方 Anthology 目录](https://aclanthology.org/venues/acl/)目前从1979年列起 | 保留历史举办清单；逐年核实是否有可索引发表材料，不能据会议举办就断言漏采正式论文集，也不据目录缺失断言未出版 |
+| COLING 1971、1976、1978 | [主办委员会历史](https://ufal.mff.cuni.cz/iccl-old)列出这三届；当前Anthology COLING目录未列出 | 来源缺口继续开放，需要主办方或出版者逐篇索引；不生成占位论文或录用/拒稿记录 |
+
+这19个位置不能用其他会议的卷补齐，也不能因本轮未找到目录而删除。公开拒稿可获得性是独立核查，不从论文集或录用清单缺失推断。
+
+## 证据与复核
+
+三份原始HTML及 `receipts.json` 保存在工作证据目录 `paper-index/nlp-history-source-review-v1/`；结构化响应URL、时间、状态、字节数和完整SHA256收录于候选JSON的 `scope_review.evidence`。三次响应均为HTTP200；未获取论文全文。
+
+| 官方来源 | 响应SHA256 |
+| --- | --- |
+| [acl-venue](https://aclanthology.org/venues/acl/) | `d04c188c0769e065f18ca9c0246f70086e2f7027ce3efe5ea0f2cdc641796b72` |
+| [coling-venue](https://aclanthology.org/venues/coling/) | `8e7ec0a2915bfbccaa26b7a7558a9cf977c4aadc119794516ecebb826f61623f` |
+| [iccl-history](https://ufal.mff.cuni.cz/iccl-old) | `c5286dc51236c90685d22543ff1f7b209adcb3eedb82a7b1ac2115c8dd0169fc` |
+
+复核时先校验保存的原始响应哈希，再读取目录中的会议身份和年度。实时网页若已变化，应另存新响应和时间，不能把新内容套用旧哈希。本次不改最初会议年度清单或生产年度快照，不代表第14步覆盖验收或第16步最终全库审计完成。

docs/coverage/iclr-archive-2014-2016.md

@@ -0,0 +1,44 @@
+# ICLR 2014–2016 官方历史录用档案
+
+此适配器是长期维护工具的一部分，按需运行。只识别已经独立核对的三种页面布局；后续页面发生变化必须重新核查，不能放松计数或轨道边界以使采集成功。
+
+| 官方页面 | 页面链接次数 | 整理后的不同记录 |
+|---|---:|---:|
+| [2014 Conference Proceedings](https://iclr.cc/archive/2014/conference-proceedings/) | 35 | main 35 |
+| [2015 accepted-main](https://iclr.cc/archive/www/doku.php%3Fid=iclr2015:accepted-main.html) | 116 | main 31、workshop 74 |
+| [2016 accepted-main](https://iclr.cc/archive/www/doku.php%3Fid=iclr2016:accepted-main.html) | 80 | main 80 |
+
+2015 的 11 篇 oral 全部再次出现在 poster 段落中。相同年份、轨道及 arXiv 标识的重复展示只生成一条来源记录，但每次展示的标题、作者显示字符串、链接、版本和段落均保留在 `source_variants`。两处标题或解析作者冲突时停止采集、等待人工核查。不同年份或不同轨道不合并投稿。
+
+2014 的作者以分号列出，一条有末尾分号，一条标题和作者斜体块不在普通段落中，部分作者段落拆成多个格式片段；必须读取完整作者段落或独立斜体块，不能只读首个格式片段。2016 的代码链接不属于作者。保留原文，解析结果不替代来源证据。oral、poster 作为 `iclr_archive:presentation` 标签，不改成独立投稿轨道。arXiv 版本留在来源变体中，稳定标识使用去版本号的编号。
+
+## 来源快照与运行
+
+2026-11-09 已冻结的官方响应 SHA256：
+
+- 2014：`a33858176fbffae0066ed3c10310f7dd98a66d12a36334b6339ef1f915ef07af`
+- 2015：`131491663742cb084662a4804009b8318e0fa7bc184d4a9044d16dd28f05b888`
+- 2016：`7320b8652431793b6ea0743f871ee446bf08b6361ccd4e421b6b53449e71f4f6`
+
+在服务器运行，输出新目录，不下载论文 PDF：
+
+```sh
+paper-atlas collect iclr-archive --year 2015 \
+  --snapshots /data/paper-index/collected/historical-next-source-evidence-v1 \
+  --out /data/paper-index/collected/iclr-archive-source-v1/2015
+
+# 或重新下载官方 HTML，并保留 HTTP 缓存与收据
+paper-atlas collect iclr-archive --year 2016 \
+  --cache /data/paper-index/cache/iclr-archive \
+  --out /data/paper-index/collected/iclr-archive-source-v1/2016
+```
+
+冻结输入目录必须包含 `receipts.json`，每项有精确 `url`、HTTP `status: 200`、相对 `file`、`bytes`、`sha256` 和带时区的 `finished_at`。读取时重新核对尺寸、哈希和路径；拒绝越界及指向目录外的软链接。已完成输出不可覆盖，应检查报告后另建明确版本目录。解析或计数失败时不生成完成报告，也不导入数据库。
+
+输出分轨道 JSONL、manifest、原始 HTML 证据和年度采集报告。仅支持这三年已审查的页面。2014 workshop、其他未列轨道、逐篇摘要及公开拒稿范围尚未通过这些页面核验，不能据此声称完整。缺席不代表 rejected、withdrawn 或从未被录用。
+
+## 导入前后的边界
+
+这三页合计 220 条不同来源记录，不等于新增 220 篇论文；同一工作可能已存在于 arXiv 或其他会议。先核验来源字段与原始页面，再执行 PostgreSQL 只读预检，比较稳定标识、标题、作者和原始来源。标题/作者差异、版本关联及冲突必须保留证据并明确评估；不能仅靠标题猜测身份，也不能覆盖原有摘要。
+
+采集器不连接数据库。正式导入使用既有 `import records`，必须有新的生产保护基线和独立验收；全部导入结束后仍需全库去重和历史覆盖审计。来自官方录用清单的 accepted 是清单观察，不是原始审稿决定签名或决定日期。

docs/coverage/ijcai-sections-2026-11-08.md

@@ -0,0 +1,40 @@
+# IJCAI 2018、2022 逐条目录与 track 核查
+
+2026-11-08 的只读生产快照中，edition 867（2018）和 871（2022）已正确属于 IJCAI/ECAI 联办会议，但整本论文集的所有投稿仍归在 `main`。本核查用官方目录的逐条 DOI 和分区定位问题，不修改生产数据。
+
+| 年份 | 库中投稿 | 官方目录条目 | DOI 一对一对应 | 官方 Main track | 明确非 main | 当前目录缺项 |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| 2018 | 870 | 870 | 870 | 709 | 161 | 0 |
+| 2022 | 863 | 862 | 862 | 678 | 184 | 1 |
+
+2018 官方分区为 Main track 709、Invited Speakers 3、Evolution of the contours of AI 6、Sister Conferences Best Papers 46、Survey track 16、Journal track 24、Early Career 19、Doctoral Consortium 17、Demos 30。
+
+2022 官方分区为 Main Track 678、Special Track on AI, the Arts and Creativity 17、Special Track on AI for Good 33、Sister Conferences Best Papers 26、Survey Track 38、Journal Track 19、Early Career 8、Doctoral Consortium 25、Demo Track 18。Special Track 保留独立范围，不因同在正式论文集就合进 main。
+
+## 862/863 差异已定位，不能删一条凑数
+
+数据库独有条目：`conf/ijcai/Fang22`，题名 **Fixed-Budget Pure Exploration in Multinomial Logit Bandits**，DOI `10.24963/ijcai.2022/409`，页码 2951–2957。submission 333599、source_record 3578445、paper 3442268。
+
+[当前官方总目录](https://ijcai.org/proceedings/2022/) 没有这一条，详情页 `/proceedings/2022/409` 本次 HTTP 404；但[官方论文文件](https://ijcai.org/proceedings/2022/0409.pdf) HEAD 返回 HTTP 200、application/pdf、153657 字节。浏览器读取的官方文件标题、作者 Boli Fang、IJCAI-22 页脚及首页 2951 与数据库对应，共七页。服务器没有下载保存 PDF 正文。
+
+因此这是已定位的“当前 HTML 目录缺项但官方文件仍存在”的来源差异。没有证据证明撤稿、拒稿或数据库多收了一篇；保留原条目与发表证据。其当前官方 track 仍待明确，不根据相邻编号擅自归 main。
+
+## 可复核证据
+
+同目录 JSON 保存两份官方 HTML 的 URL、HTTP 状态、时间、字节数和 SHA256，以及生产只读导出 SHA256。服务器快照目录 `/data/paper-index/collected/ijcai-section-audit-v1/` 保留 HTML、来源回执、原始数据库来源导出和 409 HTTP 回执。
+
+JSON 的 `years[].sections[].records` 每条为 `[submission_id, source_record_id, doi_suffix]`；DOI 为 `10.24963/ijcai.<year>/<doi_suffix>`。这些是现有稳定 ID，不是待插入的新投稿。全部 1732 个当前目录条目均对应唯一数据库来源 DOI，目录与数据库均无重复 DOI，官方目录独有条目为零。
+
+标题仅作辅助核验：NFKC、casefold、去除 Unicode 非 word 字符后，2018 全部一致；2022 有一个拼写差异，DOI `/552` 的 DBLP 标题将 Revisiting、Re-Construction 拼为 Revisting、Re-Constrcution。原始标题均保留，未因此创建或合并论文。
+
+## 未完成事项及修正边界
+
+本次交付只完成来源核查和逐条分区映射，**345 条非 main 的生产分类尚未修正**，第 14 步尚未完成。下一步需要可复用的、带快照前提和验收报告的分区修正流程：
+
+- 保持原 submission、source_record、paper、decision 的 ID 和证据内容；纠正 edition 归属，不制造“新一次投稿”。
+- 同步处理依赖 edition 的 submission_identifiers、覆盖记录与联合会议别名；不能只移动 submissions 一个字段。
+- 保存官方分区文字、URL、源文件 SHA 和原始分类，防止以后 DBLP 整卷刷新把精确 track 覆盖回 main。
+- 对 409 保持单独待核实状态，不推断撤稿、拒稿或当前 main 归属。
+- 修正后独立核验完整投稿/决定历史、稳定标识、各 track 行数及实际查询。研究方法类型不能由 track 自动生成。
+
+这是 2018、2022 两届的完整目录差集和分区证据，不代表其他年份已逐条验收，也不替代所有导入结束后的全库去重。

docs/coverage/www-1995-posters-reviewed-2026-11-09.json

@@ -0,0 +1,201 @@
+{
+  "format": 1,
+  "source_cache_sha256": "8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e",
+  "records": [
+    {
+      "entry_id": "45",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/45/index.html",
+      "individual_sha256": "83bede6afc9a7039fae2dd1f28274a394418b34f5d5632590b913617580bda1a",
+      "title": "WISE - World Wide Information System for R&D; Efforts",
+      "authors": [
+        "Fritz Loseries",
+        "Veronika Samara",
+        "Bodo Urban"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "16",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/16/index.html",
+      "individual_sha256": "cf76da903eaeaa27d19857fdaa13220f6821559bf4ac22844fae10abbaa6f3f3",
+      "title": "The World Wide Web as a Tool for Collaborative Inter-Agency Projects: Creation of an Environmental Metadata Database for the Analysis of the Venice Lagoon Ecosystem.",
+      "authors": [
+        "Mirko Labbri"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "17",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/17/index.html",
+      "individual_sha256": "0578fec9c82e8f2f98c4de0f1bcc44011ebb84c53353fdcc423a69cf70673bc2",
+      "title": "NetRepreneur",
+      "authors": [
+        "Kari-Pekka Turtiainen",
+        "Antti Auer"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "29",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/29/index.html",
+      "individual_sha256": "624ad3f685de1163411e33e90488975d848a63243fd7ffbed0862c6fb58c4140",
+      "title": "Mosaic TueV 2.4.3: Multilingual GUI and Improvements towards User Friendliness",
+      "authors": [
+        "Mike Bretz",
+        "Uwe Koch",
+        "Paul de Bra"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "13",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/13/index.html",
+      "individual_sha256": "02409321a3403a94d8362a9862f4960386ea9c344fb47f4bcdf9eae128e8b46e",
+      "title": "Using a WWW Browser as an Alternative User Interface for Interactive Applications",
+      "authors": [
+        "Frank Lonczewski"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "36",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/36/index.html",
+      "individual_sha256": "ed9dada4ea5903cb8f0a4e75f2958416a6e32647fbf3530e58edd9abf26070ce",
+      "title": "HyperSource: A Hypermedia Program Development And Documentation System",
+      "authors": [
+        "Jan Oliver Borchers"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "57",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/57/index.html",
+      "individual_sha256": "3ad4206ad28a19823172cd2845f636fe7a461c251940e9a93deb8f6484b8a984",
+      "title": "Progressive Image Transmission Capability (PROTRAC) on the World Wide Web",
+      "authors": [
+        "Vojislav Lalich-Petrich",
+        "Gaurav Bhatia",
+        "Larry Davis"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "38",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/38/index.html",
+      "individual_sha256": "940370acc5e6a954b197053fe451e466a904d1f0ac82c8fe4e6ca455faf2c927",
+      "title": "Measuring the Web with Lycos",
+      "authors": [
+        "Michael L. Mauldin"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": "Official abstract listing has no abstract. Canonical HTML has body sections but no explicit abstract; body is not substituted as a summary."
+    },
+    {
+      "entry_id": "05",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/05/index.html",
+      "individual_sha256": "1858be87d6e7f47476e1cb8fb4793c0fce3b8a6ff653bcfd4a35089e0520f177",
+      "title": "Virtual Places - Adding people to the web",
+      "authors": [
+        "Yosi Mass"
+      ],
+      "author_source": "author_index",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": "Official abstract listing and canonical HTML have no abstract. Canonical page identifies a contact and links PostScript; no full-text download. Author identity comes from author index, not inferred from contact role."
+    },
+    {
+      "entry_id": "37",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/37/index.html",
+      "individual_sha256": "b700a0a51b7a82a5f88373ae0283bb369d1cf6f455acff7d91cd7313fa8b3adf",
+      "title": "The Internet Public Library: Serving a Global Community",
+      "authors": [
+        "Susan Schweitzer"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "41",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/41/index.html",
+      "individual_sha256": "397b474109f9c1240a77d0f490f2b02dd35fae93febbb7397d1b2a31759a8ec1",
+      "title": "Graphical Browsing in the WorldWideWeb and the MultiMedia Forum",
+      "authors": [
+        "Rainer Pagé",
+        "Klaus Süllow"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "47",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/47/index.html",
+      "individual_sha256": "52eb4bfdd658730a222c98170b72dabf9cbcc5ffd2ada1e900b826baf2b02833",
+      "title": "Interoperating Advanced Hypermedia Systems with WWW",
+      "authors": [
+        "Jian Zhang",
+        "Martin Fruehauf",
+        "Thomas Kirste"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "50",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/50/index.html",
+      "individual_sha256": "8dc8b0561bd8b1b106c5b209a973c695c3cc9df5288450d4a01fe32541476b7e",
+      "title": "Web Enabled Agile Manufacturing",
+      "authors": [
+        "S. V. Iyer",
+        "T. Singh",
+        "U. Ravaioli",
+        "R. E. DeVor",
+        "S. G. Kapoor",
+        "G. Veknatasubramian"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "55",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/55/index.html",
+      "individual_sha256": "21780cd4047f513e8c9aeced5247002e27cf71719f08de85c6531b57a9e5058d",
+      "title": "Weblets: Fundamental Building Blocks for WWW Tools",
+      "authors": [
+        "Thomas J. Watt Jr."
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    },
+    {
+      "entry_id": "59",
+      "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/59/index.html",
+      "individual_sha256": "fed4cc9dcaee350a12c5e4b6fa9c99be9bead25a81632512d31566400d3d0fad",
+      "title": "Usenet and the World-Wide Web",
+      "authors": [
+        "M. L. Grant"
+      ],
+      "author_source": "canonical_byline",
+      "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
+      "missing_abstract_reason": ""
+    }
+  ]
+}

docs/data-sources.md

@@ -95,6 +95,12 @@ paper-atlas audit public-openreview --db /data/paper-index/papers.sqlite \
 
 输出按会议、届次及 main、position、datasets_benchmarks 分文件。workshop 默认进入单独的 held 清单；有明确 workshop 名称时可通过 `--include-workshops` 单列采集，不合并到主会。Retraction、未映射的轨道或负面标签也保留在 `held-records.jsonl`，不强改为 Reject 或 Withdrawn。当前决定表没有独立 retracted 状态，因此撤回发表记录仍需后续专门建模。
 
+`parser_version=2` 还会将明确负面标签但标题缺失、空白或 null 的记录放入 `held-records.jsonl`，保留原始状态、forum ID、固定来源版本和行号；不制造标题或导入不可识别的论文，也不让这一条缺失阻断其他可用记录。held 文件哈希与各原因数量保存在 inventory；非文本标题、错误身份、冲突届次和哈希不符仍使整批失败。旧年 ICLR 镜像的明确 `Withdraw` 标签归为 `withdrawn`，`Workshop`、`Poster Workshop`、`Oral Workshop` 均不作为拒稿。采集文件与导入主库仍是两个独立步骤。
+
+`parser_version=3` 增加五组已核实身份的 ICLR workshop 标签：2024 ME-FoMo / GenAI4DM 的 Withdrawn Submission、2025 AgenticAI 的 Reject / LMRL 的 Withdrawn Submission，以及 2026 LLM Reasoning 的 Withdrawn Submission。仅在 `--include-workshops` 下输出，并分别保存 `workshop_me_fomo`、`workshop_genai4dm`、`workshop_agenticai`、`workshop_lmrl`、`workshop_llm_reasoning`；默认仍暂存。标签必须与原文件会议、年份和原始 workshop track 一致，错年/错会议使采集失败，错轨道单独暂存，未知标签继续待核查，不用宽泛的 Reject/Withdraw 正则猜测。
+
+[OpenReview 官方 ICLR 活动目录](https://openreview.net/venue?id=ICLR.cc) 确认上述独立 workshop 身份；[ME-FoMo 2024](https://sites.google.com/view/me-fomo2024/call-for-papers)、[GenAI4DM 2024](https://sites.google.com/view/genai4dm-iclr2024)、[AgenticAI 2025](https://iclragenticai.github.io/) 和 [LLM Reasoning 2026](https://sites.google.com/view/iclr-2026-llmreasoning) 的活动网站也提供对应投稿入口。输出在 metadata 保存 `workshop_venueid` 和身份目录证据，决定仍只标为固定镜像的公开分类断言，保留原始标签、forum、来源行号与哈希，不声称核验原始决定签名。2026-11-08 核查时部分原站记录页要求浏览器验证；身份目录和主办方入口可读，不代表逐篇原始决定已可访问，也不外推其他 workshop 的公开政策。采集后仍须经过 PostgreSQL 身份/决定冲突预检、串行导入和独立验收。
+
 该来源明确标记为 `curated_public_status_assertion`：保留源 commit、文件哈希、行号、原标签和 forum ID，不声称已核验原始 PC 签名。只保存标题、摘要、公开作者名、关键词、领域、索引与决定来源，不缓存镜像中的个人资料、邮箱、性别、评审内容或 PDF。所有输出清单均为 partial；盘点中的零条记录仅描述这个固定镜像。
 
 入库前审计核验文件哈希、数量、轨道和来源声明，并用真实 forum ID 对比同届同轨道已有投稿。存在不同决定时阻止整批导入，交由来源和时序核查；另一会议的后续录用属于独立投稿历史，不构成该次拒稿的冲突。同一 forum 的镜像不新增投稿事件，但论文实体的跨平台去重仍须在全量结束后核验。审计通过的报告包含带 SHA-256 的导入队列；应在单写入维护锁下及时串行导入，主库变化后重新审计。
@@ -180,6 +186,10 @@ paper-atlas collect dblp \
 
 输出按会议/年份/track 分文件，附 manifest、`dblp-normalization.json` 范围审查统计和带哈希的 `dblp-import-queue.json`。不能识别的书目、年份或归属会逐类登记，不默认算作主会。DBLP 不含摘要，因此输出摘要为空、清单为 partial；需要通过有证据的跨来源关联补齐论文实体摘要。书目发表证据与原始审稿决定分别保留，不依据书目缺席推断拒稿。已存在其他出版来源的届次，还须先回填其权威身份并核验镜像记录，不能直接累加投稿数。
 
+早期 AISTATS 1995/1997/1999/2001/2003/2005 和 COLT 1988–1991 的主系列名称本身含 Workshop，不能当作附属 workshop 漏掉。识别要求已核对的完整 proceedings key、会议书目名、原始标题前缀、届数和事件年一致；轨道名为 `main` 表示主系列，保留原始 Workshop 名称及官方目录证据，不改造其历史名称或出版形式。参见 [AISTATS 官方历史目录](https://aistats.org/other.html)、[COLT 协会目录](https://www.learningtheory.org/)。1995 AISTATS 的会议年与1996图书出版年分开；[PMLR R0](https://proceedings.mlr.press/r0/) 是后来重发的 pre-proceedings，不与 DBLP 图书条目盲目逐篇等同。
+
+真正附属的 AISTATS 2010 Active Learning and Experimental Design workshop 仍单列。COLT 2003 与 Kernel Workshop 的 [混合论文集](https://doi.org/10.1007/b12006) 缺少逐篇章节信息，使用 `proceedings_unspecified`，不能整卷改成 main 或附属 workshop。输出保存 `track_scope_note` 与 `track_scope_evidence_url`，导入后仍可追查分类理由；这些出版证据不是真实拒稿或原始决定签名，也不自动修正既有主库记录。
+
 部分早期 SIGIR、CIKM、IJCAI 出版卷使用两位年份键；只有论文集标题明确给出唯一完整年份且与短键相符时才采用，不推测世纪。WWW 特殊兴趣、Alternate track 单列；同卷混合 Applied Data Science、Demo、Industrial 且不能逐篇区分时使用 `proceedings_unspecified`，不把整卷归为某一个具体 track。
 
 书目中的前后页、目录组合和明确会务标题不产生论文录用记录；缺作者的赞助、委员会、活动汇总等标题单独排除。WWW 2019 companion 的匿名合成活动条目归入待核查材料，原始快照仍保留。范围报告分别统计这些原因；单纯缺作者并不排除真实研究论文，也不补造作者。

docs/edition-tracks.md

@@ -0,0 +1,45 @@
+# 按证据修正既有投稿的轨道
+
+`paper-atlas maintain edition-tracks` 用于已导入整卷论文集、后来取得逐条官方分区证据的情形。它更正原投稿的 edition 归属，保留投稿 ID、所有决定和来源关系；不会重新导入论文、合并实体、生成研究方法标签或推断拒稿。支持独立 SQLite 样本和共享 PostgreSQL；不得写入已冻结的生产 SQLite 备份。
+
+## 证据与完整范围
+
+先核实官方目录、稳定 DOI、原始来源记录和分区，保存原始页面及 SHA256。输入 JSON 为 `{"format":1,"records":[...]}`，每条必须包含：
+
+| 字段 | 含义 |
+| --- | --- |
+| `submission_id`、`source_record_id`、`external_id` | 原有投稿、主来源记录及主来源投稿标识 |
+| `source`、`doi` | 原来源名称及其 metadata.record_identifiers 中的 DOI |
+| `from_edition_id`、`year`、`venue` | 原 edition 及其真实年份和规范会议名 |
+| `venue_names` | 规范会议名加全部已有别名；必须与库中一致 |
+| `track` | 经核实的目标轨道，使用小写字母、数字、下划线；范围未明确可用 proceedings_unspecified |
+| `section` | 官方分区原文或明确的未确定说明，不推断研究方法 |
+| `evidence_url`、`evidence_sha256` | 已人工/独立核实的 HTTPS 原始证据和内容哈希 |
+
+输入必须列出每个原 edition 的**全部**投稿，包含留在原轨道的记录。工具核对身份、DOI、单来源发表观察、现有别名、完整范围以及目标冲突；不自动解释 HTML，也不把填写了 URL/哈希当成已证明内容。核查者应先独立证明每条分区对应关系。
+
+多来源共同拥有的投稿、有非 accepted 决定历史的投稿、已有分类修订、非空目标 edition、目标已有覆盖或标识等情形会停止，需要专门核对。每个原 edition 须有一个明确来源清单；其他来源或多清单覆盖需先另行协调。本功能不是任意批量移动或跨会合并工具。
+
+## 只读计划、事务执行和验收
+
+```sh
+paper-atlas maintain edition-tracks \
+  --pg-service foundry-research --pg-schema research \
+  --inventory reviewed-inventory.json --out plan.json
+
+sha256sum plan.json
+
+paper-atlas maintain edition-tracks \
+  --pg-service foundry-research --pg-schema research \
+  --plan plan.json --plan-sha256 <上一步的完整SHA256> --out applied.json
+```
+
+报告路径必须是已有目录中的新文件。预检只读。执行使用 PostgreSQL 单写入 advisory lock，或 SQLite BEGIN IMMEDIATE；整个分类修正、逐来源证据、标识、覆盖及验收在一个事务中完成。计划绑定数据库身份、全部来源文本/非修订元数据、标签、可选 annotation_events、投稿/决定/来源/标识内容及原覆盖快照；过期即停止。失败回滚数据，PostgreSQL 序列可能留下正常间隙，不能用 max(id) 当数量。
+
+目标 edition 沿用原会议、年份、event_key，不删除或重命名原 edition。只更正既有 submissions.edition_id 和对应 submission_identifiers.edition_id。原文、来源检索时间、决定 ID/状态/证据及各表来源关系不变。
+
+分类证据写入来源元数据的保留字段 `edition_scope_curation_v1`，包括原范围、目标 edition、官方分区、URL、哈希和计划摘要。普通来源导入不能伪造或替换此字段；后续 record 刷新保留修订。相同来源投稿通过旧范围调用 decision 时仍路由到修正后的 edition。旧整卷 main 文件通过 import records 刷新会报错并回滚，要求先按核实轨道重新整理，避免覆盖正确的统计。
+
+覆盖保留原清单 URL、cursor、成功时间，按实际目标分配 expected_count/imported_count 并标 partial；本工具核实分类，未核实摘要或所有来源完整性。原覆盖整行保留在计划中，不能继续将整卷数当 main 论文数。应用后校验原 edition 身份、全部受保护内容、分类、标识、修订字段和覆盖；相同计划再次执行仅在这些验收条件仍成立时返回 already_applied。
+
+生产使用前必须在已确认 runner 完成软件验证。正式数据修正后仍需独立验收，并在最终全库审计中复核；单元测试和应用回执不能代替生产验收。

docs/event-identity.md

@@ -0,0 +1,44 @@
+# 同年多届会议的事件身份
+
+`venue`、`year`、`track` 描述系列、年份和投稿轨道；`event_key` 区分同系列、同年、同轨道的实际会议事件。例如 WWW 1995 的春季与秋季会议分别使用 `www-1995-spring` 和 `www-1995-fall`，两者仍是 `main`。相同论文可以在两届有不同投稿与决定；论文去重不会把这两次投稿合成一次。
+
+普通单届数据继续使用空键。新输入在同一组已有具名事件或已知同年多届时必须提供明确键；不能任选第一个 edition。已有来源刷新可依据自己的原投稿关系回到原事件，不能被显式改到另一事件。精确固定 DBLP 快照、WWW Fall 书目名、1995 年及 `www1995f.html` 卷共同支持旧秋季文件的归属；不按标题猜年份或季节。
+
+规范 JSONL 行及 manifest 都可提供 `event_key`，非空值限制为最多80个小写字母、数字及 `._:-`，首字符为字母或数字。manifest 提供键时，每行必须一致；一份文件仍仅对应一个事件及轨道。跨行冲突会回滚整份业务导入。身份回填同样按事件选择，不能借机把来源移到另一届。
+
+## 新样本与既有数据
+
+新建 SQLite 样本的 `editions` 唯一键为 `(venue_id, year, track, event_key)`。已有 SQLite/PostgreSQL 不会因普通读写命令自动升级。其普通单届导入继续可用；WWW1994/1995或带事件键的导入会要求先迁移。冻结的 SQLite 生产备份仍受 cutover 标记保护，不能写入或升级。
+
+先在 PostgreSQL 生成只读计划：
+
+```sh
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas maintain event-identity \
+  --pg-service foundry-research --pg-schema research --out /data/paper-index/event-plan.json
+```
+
+检查计划的数据库身份、原 edition IDs、业务表数量、目标来源与历史指纹。计数也包含已显式启用的可选注解表。精确计数在同一快照中按主键分页，每页最多5000行，只读取主键；不使用估计行数。CLI向stderr持续报告计数进度，避免大库长时间无输出。已有未标记 WWW1995 edition 必须逐投稿都有固定 DBLP Fall 卷证据，才会只为原 edition 填入 `www-1995-fall`；缺证或混合卷会拒绝计划。本文命令不表示生产已经迁移。
+
+应用时传入同一文件及命令输出的文件 SHA256：
+
+```sh
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas maintain event-identity \
+  --pg-service foundry-research --pg-schema research \
+  --apply-plan /data/paper-index/event-plan.json --plan-sha256 PLAN_FILE_SHA256 \
+  --out /data/paper-index/event-applied.json
+```
+
+原生 PostgreSQL writer 使用现有 schema advisory lock，并在事务中锁定 editions，核对计划后添加列及替换唯一约束。SQLite 样本使用独占写事务，重建父表时保留 IDs，并在提交前检查外键；不级联删除子记录。额外列、未知唯一约束或 SQLite 额外索引会拒绝自动升级，须单独评估。未改变原版四列的排序内容哈希、任何业务表数量或目标投稿/决定/来源内容，才可提交；失败回滚。迁移标记与 DDL 同事务。原计划立即重试为 `already_applied`；若后续业务已经变化，应重新生成计划，不重放旧计划。
+
+ECML 与 PKDD 的联合举办前事件已有不同 venue/edition，保持原身份即可。这项迁移不会把它们合并，也不会创建新的论文、投稿或决定。WWW 春季及1994两届的逐篇采集、来源核验和导入仍须另做。
+
+## 查询与验收
+
+```sh
+paper-atlas query --pg-service foundry-research --pg-schema research \
+  --venue WWW --year 1995 --track main --event-key www-1995-fall
+```
+
+详细投稿历史及 `known_acceptances` 返回 `event_key`。`--event-key` 可与 `--status` 一起限定同一事件中的最新已知决定。未迁移的旧库仍可普通查询，其事件键为空；不能声称已经区分季节。`audit histories` 的新库事件输出也保留键。
+
+远程测试应覆盖同一作品的两届独立投稿、拒稿后另一届录用、原 IDs 与 Unicode/NUL 来源内容保留、旧文件刷新、歧义及错误键拒绝、整文件回滚、明确事件的身份回填、旧计划失效、重试与真实 PostgreSQL 迁移。生产应用后还须独立核验约束、来源和历史、全文索引及 foundry-core 健康；单元测试不替代生产或最终全库去重验收。

docs/joint-editions.md

@@ -0,0 +1,67 @@
+# Repairing joint conference membership
+
+EURAI's [official history](https://www.eurai.org/ecai/history) identifies ECAI 2018
+and 2022 as joint IJCAI/ECAI conferences. The pinned DBLP volumes
+`conf/ijcai/2018` and `conf/ijcai/2022` use `IJCAI` as their booktitle and title.
+Existing records can therefore already be present under IJCAI while an ECAI query
+shows a missing year. Adding ECAI as a global alias to the ordinary IJCAI venue
+would incorrectly include all other IJCAI years.
+
+The existing canonical `ECAI+IJCAI` venue has both member aliases. This maintenance
+command changes only the venue reference of the two existing `main` edition rows.
+Edition IDs, submission IDs, decisions, source text, authors, identifiers, tags,
+source links and coverage records stay intact. It does not merge or create papers
+or submissions, resolve per-paper tracks, or turn reviews into negative decisions.
+The DBLP collector applies the joint mapping only to these two exact main volume
+keys; original booktitle/title remain in source metadata with the membership evidence.
+The shared ingestion entry point also resolves these exact pinned source volumes before
+creating an edition, so refreshing an older normalized file cannot recreate a separate
+IJCAI edition. Other sources, years, workshop volumes and additional tracks retain their existing rules.
+
+## Review a read-only plan
+
+Use a runtime containing this repair command. The configured PostgreSQL writer lock
+is used only for the apply operation; do not replace a newer ingestion runtime with
+an older main-based feature branch.
+
+```sh
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas maintain joint-editions \
+  --pg-service foundry-research --pg-schema research --out joint-plan.json
+```
+
+The preflight requires the existing joint venue and exactly its two member aliases,
+one nonempty unambiguous edition per year, and matching pinned DBLP volume evidence
+for every existing submission. A colliding target edition, missing provenance or
+unexpected source scope fails closed. The plan binds current database/schema, IDs,
+counts and full fingerprints of affected papers, identifiers, source records, tags,
+submissions, decisions and their provenance, and coverage rows. Observation time is
+recorded separately from conference year. These are scoped checks, not a final global
+identity or conference completeness audit.
+
+## Apply the exact reviewed plan
+
+Compute the file's SHA256 after review and pass it explicitly:
+
+```sh
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas maintain joint-editions \
+  --pg-service foundry-research --pg-schema research --plan joint-plan.json \
+  --plan-sha256 "$PLAN_SHA256" --out joint-applied.json
+```
+
+The plan hash is checked before opening a writer. Current evidence is checked again
+under the dataset writer lock; both edition updates and an audit marker commit in one
+transaction. Protected row hashes are checked again before commit. Changed evidence
+requires a fresh reviewed plan. Existing output reports are preserved; choose a new
+path for another run. SQLite samples use `--db sample.sqlite`; the frozen production
+backup remains protected.
+
+If a response is lost after commit, rerun with the same plan and a new output path.
+When protected evidence is unchanged and both editions already point to the joint
+venue, the result is `already_joint` with zero updates. It does not replay imports or
+regenerate source records. Keep the plan, apply report and independent postcheck
+alongside collection evidence. The audit marker is `joint_ijcai_ecai_2018_2022_v1`.
+
+Publication counts can differ between the DBLP volume and the current publisher
+page. This membership repair preserves all existing rows rather than deleting entries
+to force an aggregate match; section-level main/invited/other classification and
+publication completeness require separate per-paper evidence.

docs/operations.md

@@ -97,13 +97,16 @@ paper-atlas audit arxiv-tags --db /data/paper-index/papers.sqlite \
 采集文件、已入库的来源记录和去重论文实体是不同的计数。来源可能覆盖同一篇论文，同一篇论文也可能有多次投稿；不能把各批文件数量相加作为独立论文总数。实时统计与覆盖报告使用：
 
 ```sh
-paper-atlas query --db /data/paper-index/papers.sqlite --stats
-paper-atlas coverage --db /data/paper-index/papers.sqlite \
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --pg-schema research --stats
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
+  --pg-service foundry-research --pg-schema research \
   --out /data/paper-index/collected/venue-coverage-current.json
 ```
 
 manifest 和采集器状态清单用于确认文件是否齐全及是否缺摘要；`ingest_runs` 和协调器日志用于确认主库导入是否成功。官方程序、书目目录和公开投稿快照分别保留，未出现在某份清单不等于被拒稿；第三方快照也不能代表平台当前全量状态。代码测试成功不能代替最终全库去重、完整性和逐届覆盖核验。
 
+`coverage` 的所有计数来自同一个只读快照。`latest_observed_status_counts` 按每次投稿的最新观察决定统计，`decision_observation_counts` 保留所有历史观察；重复来源不增加投稿数，缺少决定的投稿计入最新状态 unknown，但不凭空制造 unknown 决定。`source_record_counts` 统计来源在该届次的实际记录与缺摘要数，和 `sources` 中逐清单的预期/导入数量分开；身份补充清单与原始论文集清单可以描述同一条记录。来源缺摘要与论文实体从其他来源获得摘要也分别报告。报告仍将历史覆盖标为待独立审计，不能用某个清单 complete 或没有拒稿记录推断全会议覆盖。
+
 当前代码与合并状态以 [Pull Requests](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls) 为准；CI 结果以对应提交的 Actions 检查为准。工作分支交付后等待用户手动合并。合并代码不自动部署、不自动启动采集，也不代表数据批次已完成。
 ## 修正已有 PMLR 投稿的轨道
 
@@ -120,3 +123,36 @@ paper-atlas maintain pmlr-tracks --db /data/paper-index/papers.sqlite \
 检查报告后，在同一命令添加 `--apply` 并使用新的 `--out` 路径执行迁移。计划使用共享维护锁，实际迁移使用独占锁；已有 arXiv 类别修复或其他写入运行时会拒绝启动，不会停止或重启它们。一次事务内核对整个来源集合、原始题目/摘要/作者、forum 身份、已有事件及目标冲突，更新原投稿的 edition 和身份归属，回填官方轨道出处及逐轨道覆盖数，最后复查轨道与外键后提交。论文、来源、投稿、决定的 ID 及决定观察均保留，不重新创建事件、不做实体合并、不改论文正文元数据或标签。
 
 报告记录每个迁移事件的旧 edition、原身份、完整决定观察及新轨道证据。`planned` 是未执行的计划；`complete` 才表示事务已提交；`error` 表示执行失败。若进程在提交前后中断而报告停在 `prepared`，先对照主库和报告核查，不根据日志缺失自行重启任务。新路径重跑会重新核对当前数据库，已正确的轨道不会再次迁移。共享多个来源、重复目标事件或冲突决定会停止迁移，需另行核对，不能直接删掉其中一条。
+
+## 追加同一目录的部分记录
+
+默认 `import records` 把 manifest 当成一份来源快照，覆盖统计使用该文件的数量。
+若目录已有2条，之后仅补入1条，不能把这一小批当成完整快照，否则会把目录计数改成1。
+为已核验的部分补充批次，在 manifest 显式声明：
+
+```json
+{
+  "coverage_mode": "incremental",
+  "coverage_previous_count": 2,
+  "coverage_previous_expected_count": 2,
+  "coverage_expected_count": 4,
+  "status": "partial",
+  "count": 1
+}
+```
+
+这只是新增字段示例，manifest仍须保留source、venue、year、track、list_url、文件哈希等来源证据。
+`count`是本文件行数，`coverage_previous_count`是导入前该来源/届次的实际记录数，
+`coverage_previous_expected_count`是旧覆盖行的expected_count（未知时用null），
+`coverage_expected_count`是独立目录证据支持的整份来源预期数，不能小于已知旧预期数。
+这个例子成功后覆盖统计为预期4、已导入3；文件checkpoint的records_written仍为1。
+刷新相同记录不重复累加。初次导入部分来源可声明previous_count为0、previous_expected_count为null。
+
+只支持当前最多一个规范目录URL且旧覆盖状态为partial的来源/届次；多目录、旧状态complete、
+旧计数与实际来源记录不符，或前置计数已变化，都会中止，先核对来源范围或重新取得完整快照。
+该模式在写入前检查父状态，并在同一文件事务中按实际来源记录重新计数、检查总量界限和乐观更新覆盖行。
+失败会回滚本文件的论文、来源、决定、标签和覆盖变化，保留失败运行记录；更早已提交文件不重放。
+增量模式始终保留partial，不能靠一个小批次推断完整论文集、摘要或公开拒稿覆盖。
+
+仍需先独立核对新增记录身份与决定，按原生单写入保护串行导入，再独立核验数据与原有历史。
+这些计数检查不证明外部目录完整，也不代替最终全库去重。

docs/research-runtime.md

@@ -0,0 +1,49 @@
+# Integrated research runtime
+
+This branch composes the independently reviewed identity guards (PR25), incremental
+coverage (PR27), joint-edition refresh guard (PR29), event identity (PR30), and optional
+annotation support (PR28). It starts from main 1b81d1a and preserves their commits;
+it is an explicit integration dependency, not evidence that those PRs have merged.
+Users merge through Gitea manually. Review this combined diff against main, including
+its conflict resolutions, before using it as the maintained runtime.
+
+The shared Store preserves stable source/identifier anchors before title matching,
+rejects ambiguous unanchored matches, resolves pinned IJCAI/ECAI volumes before event
+lookup, and selects same-year events by event_key. Incremental manifests for a named
+or ambiguous event must state event_key; their parent count and optimistic update
+are checked against that exact event, not the first venue/year/track match.
+
+Query filters compose event_key, status, tags and research type. An earlier rejected
+attempt and a later known acceptance remain distinct; neither annotations nor track
+names infer a research type. Installing this code does not initialize annotation_events.
+Event plans count that optional table when present, alongside the original business
+rows. Annotation initialization and event identity upgrade remain explicit commands
+with their own transactional checks. Neither command imports historical records.
+
+## Production sequencing
+
+1. Verify the exact integrated commit on the approved ces-control runner, including
+   the real PostgreSQL interaction tests and wheel/installed CLI checks.
+2. Stage that exact source in a separate release directory and verify package hashes;
+   leave the current runtime and frozen SQLite backup intact.
+3. Inspect live writers and coordinate through the existing database writer lock.
+   Generate a fresh read-only event plan using that integrated release. Check its
+   database/schema identity, all business counts and preserved history hashes. A plan
+   from before further data or optional-schema changes must not be reused.
+4. Apply only that reviewed file with its actual SHA256, in one transaction. Independently
+   verify the constraint, original edition IDs and fields, assigned Fall source, source
+   and history bytes, ordinary queries, full-text queries and foundry health.
+5. Use the verified integrated release for subsequent writes. Older writers lacking
+   event identity support must not resume against the upgraded schema. Annotation
+   initialization requires a separate explicit operation and acceptance.
+
+These are operational requirements, not claims of completed production deployment.
+The production event-key migration and optional annotation activation have not occurred
+at preparation time. Existing migration/import/repair acceptance remains valid; do not
+rerun those jobs to install source code. Cross-feature tests use disposable fixtures
+and PostgreSQL schemas, never the production research tables.
+
+Coverage output also returns event_key for each independent edition ID. Two same-year
+main events keep their own status/source counts; venue paper totals remain distinct
+work counts rather than sums across events. Legacy datasets without the event column
+return an empty event key and remain read-only compatible.

docs/rss-proceedings.md

@@ -0,0 +1,38 @@
+# RSS 官方论文元数据
+
+这是长期维护、按需运行的 package collector，不是定时服务或一次性导入脚本。目前仅接受独立核对过的 2025 年版式；不把 2026 年或其他年份套用同一分类规则。
+
+## 数据范围和证据
+
+- [RSS XXI 官方论文集](https://www.roboticsproceedings.org/rss21/)列出 163 个不同的逐篇 HTML 链接。每篇有 citation 元数据、BibTeX 和摘要区；PDF 链接作为原始元数据保存，不请求全文。
+- [官方 Accepted Papers](https://roboticsconference.org/2025/program/papers/)按论文 ID、标题和 session 对账。网页作者列包含隐藏的重复展示，逐篇有序 citation_author 与论文集目录作者完整对比后才采用；不截断作者、不将重复展示当新论文。
+- [官方 Demos](https://roboticsconference.org/2025/program/demos/)明确列出 15 个论文 ID，包括标题不以 Demonstrating 开头的 ID 93。这些条目标为 `demo`，保留整行节目证据。其他 148 条使用 `proceedings_unspecified`：没有逐篇类型依据时，不因缺席演示表就自动推断 main。
+- [作者说明](https://roboticsconference.org/2025/information/authorinfo/)描述 Science/Systems 与 Demo 两类投稿，两类都进入论文集。分类属于该来源的投稿/发表语义，不自动生成通用 research_type 标签。
+
+`rss_proceedings:session` 保留官方 session，`rss_proceedings:program_role` 保留明确的 demonstration。二者与其他平台标签隔离；session 主题不等于投稿 track。单篇来源 ID 是 `2025/pNNN`，DOI 同时保留原始 BibTeX 大小写和标准化身份键，显式的 `publication_doi` 供原生导入器在同一会议年度/track 中关联其他正式发表来源；不能据此合并不同 track 或不同会议的投稿经历。
+
+只有已发表、被官方录用表列出的条目才产生 accepted 记录。采集观察时间来自响应回执；出版日期单独留在 metadata，不伪造原始决定日期。来源未列出的论文不能推断为 rejected、desk rejected 或 withdrawn；公开负面决定是另一个待核范围。
+
+## 运行
+
+构建、安装和项目测试仅使用已确认的 `ces-control:host` 远程 runner。本机不能执行隐式构建的验证。服务器已安装环境运行示例：
+
+```sh
+paper-atlas collect rss-proceedings --year 2025 \
+  --snapshots /data/paper-index/collected/rss2025-full-evidence-v1 \
+  --out /data/paper-index/collected/rss2025-source-v1
+```
+
+也可使用 `--cache /data/paper-index/raw/rss-http` 请求限定官方 HTML 元数据。两种输入互斥。冻结快照的 `receipts.json` 每项必须包含 `file`、准确 `url/final_url`、`http_status: 200`、`content_type: text/html`、`bytes`、`sha256` 和含时区的观察时间 `at`。路径必须位于快照内，不能通过符号链接逃逸。所有输入逐个核验字节数与哈希，UTF-8 解码；重定向、未知版式和异常 ID 需要先核查。
+
+采集只输出 JSONL、每个 track 的 manifest、完整 HTML 证据和 `rss-2025.json` 来源报告，不连接生产数据库。两份清单状态保持 `partial`；即使该卷 163 个链接全部解析，也不表示独立作品去重、逐篇类型或公开负面决定全集已完成。明确为空的摘要保存 `null` 和原因；缺失或重复摘要边界属于版式异常，停止输出。
+
+## 冻结与恢复
+
+2026-11-09 在服务器冻结 167 个官方 HTML 响应：目录、录用表、演示表、作者说明和 163 个逐篇页面。回执 SHA256：`f427d93a113702030864e789efe1176d69f71d2c567e81536d155e53d9513a93`。六个未改写样例位于 `tests/fixtures/rss2025/`，附其真实响应回执；完整原件保存在服务器来源目录与独立审阅备份，不作为运行时下载依赖。
+
+整个采集持有输出目录的排他锁。全部解析和交叉检查通过后才发布 JSONL、manifest 和完成报告；完成报告最后发布。已有完成报告或 JSONL 时拒绝覆盖。若发布中断，先核对目录、回执和所有文件哈希，保留现场，再换新输出目录重采；不能看到日志安静就重启或重复导入。新的官方页面内容或计数与冻结样本不同，必须另留快照、解释变化并验证后再更新采集器。
+
+生产导入前必须另外做来源独立核验和 PostgreSQL 只读身份预检。DOI、同标题候选和既有 arXiv 实体需要逐项核查，保留跨会的所有投稿和决定关系；发现冲突时隔离，不能只按标题强合并。已导入队列不得重放。生产独立验收通过后才重新统计年度缺口；导入成功和 CI 不能替代最终全库去重、摘要/标签及约束和实际查询审计。
+
+本说明中的 163/15 是冻结官方页面的来源清单，不是当前生产库导入结果。历史缺口在本批生产验收前仍为 47。

docs/usage.md

@@ -2,6 +2,55 @@
 
 先在已确认的远程环境安装项目，使用 `paper-atlas`。从源码运行时，将命令前缀替换为 `python3 -B -m paper_atlas`，其余参数相同。数据目录位于仓库外。
 
+## 按证据补充稳定标识
+
+`paper-atlas curate identifiers --plan /data/review/plan.json --output /data/review/prepared`
+只生成新的 JSONL、manifest、证据副本及 `curation-report.json`，不会连接数据库。
+输出目录必须不存在，父目录必须已存在；任何输入或断言失败，都不发布本批输出。
+原始文件不变，未选中的记录逐字节保留。题名、作者、年份、页码、投稿决定、标签和既有 metadata
+均不重写，仅补充 `identifiers` 并追加 `metadata.identifier_curation` 证据链。
+
+计划格式如下（哈希占位符须替换为真实 SHA256，所有路径按命令当前目录解析）：
+
+```json
+{
+  "format_version": 1,
+  "inputs": [{
+    "file": "/data/review/uai2001.jsonl",
+    "sha256": "<JSONL 文件 SHA256>",
+    "manifest_sha256": "<同名 .manifest.json 文件 SHA256>"
+  }],
+  "assertions": [{
+    "source": "dblp",
+    "external_id": "conf/uai/example",
+    "record_sha256": "<原始物理行字节 SHA256，包含该行原有换行符>",
+    "identifiers": {"arxiv": "1408.1485"},
+    "evidence": [{
+      "file": "/data/review/primary-metadata.json",
+      "sha256": "<证据 JSON 文件 SHA256>",
+      "url": "https://example.org/primary-record",
+      "checked_at": "2026-11-08T08:00:00Z",
+      "note": "说明人工核对的题名、作者、出版版本和稳定标识证据"
+    }]
+  }]
+}
+```
+
+输入 manifest 必须包含 `source/venue/year/track/count/list_url/file_sha256`，且与逐条记录一致。
+断言必须唯一、准确命中一条记录。仅支持无版本后缀的 arXiv 编号与小写规范 DOI；
+已有同类标识不允许覆盖，相同值可以补记证据。证据只接受不超过 16 MiB 的 JSON 元数据文件，
+来源 URL 要求 HTTPS 且不含凭据，核查时间必须带时区；不下载论文 PDF。
+
+`verification_scope=input_and_evidence_integrity_only` 和 `semantic_identity_verified=false`
+明确表示哈希校验不能证明两条记录属于同一作品。请先独立核对版本关系；仅年份、页码或相同题名
+不能作为合并依据。存在来源冲突的行应继续暂缓。新 manifest 保留原来的 partial 状态及摘要缺失数，
+不会把补充编号伪装成来源覆盖或摘要补全。证据副本按 SHA256 保存在输出目录 `evidence/`，
+原计划也保存在 `plan.json`，供之后复查。
+
+文件生成后仍需另做当前 PostgreSQL 身份、投稿和决定冲突预检；导入运行时必须包含
+“稳定标识优先、多个标题候选则整文件回滚”的身份保护，再以 `import records` 独立导入并验收。
+该命令不会自动导入或合并实体，也不代表最终全库去重通过。
+
 ## PostgreSQL 查询
 
 在远程环境安装 `.[postgres]`，通过权限为 `0600` 的 libpq service 文件连接，不把密码放入 URL、命令行或仓库。
@@ -79,22 +128,43 @@ paper-atlas maintain merge --db /data/paper-index/samples/example.sqlite \
 ## 4. 查询与分析前检查
 
 ```sh
-paper-atlas query --db /data/paper-index/papers.sqlite --query '"language model"' --limit 20
-paper-atlas query --db /data/paper-index/papers.sqlite --tag arxiv:category=cs.AI
-paper-atlas query --db /data/paper-index/papers.sqlite --venue ICLR --year 2025 --status rejected
-paper-atlas query --db /data/paper-index/papers.sqlite --paper 123
-paper-atlas coverage --db /data/paper-index/papers.sqlite --out /data/paper-index/collected/coverage.json
+paper-atlas query --pg-service foundry-research --query '"language model"' --limit 20
+paper-atlas query --pg-service foundry-research --tag arxiv:category=cs.AI
+paper-atlas query --pg-service foundry-research --venue ICLR --year 2025 --status rejected
+paper-atlas query --pg-service foundry-research --paper 123
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
+  --pg-service foundry-research --pg-schema research --out /data/paper-index/collected/coverage.json
 ```
 
 `--status rejected` 筛选某次投稿的最新已知决定。它并不表示论文在所有会议均未录用；分析转投情况时查看该论文的完整投稿历史。年份表示会议届次，观察时间和决定日期分别保留。
 
+覆盖报告使用只读一致性快照，标明实际数据库和 schema，并列出零入库目标与注册表外的已入库会议。逐届、逐 track 分别列出最新观察状态的投稿数、全部决定观察数、各来源记录数，以及来源摘要和论文实体摘要的缺失数。摘要缺失指 NULL 或空字符串；有文字不代表内容已经完整核验。同一来源的多个清单 URL 可以重叠，不能把清单计数相加当作来源总量；零条拒稿也不证明该会议没有公开拒稿。SQLite 样本仍可显式使用 `--db`，生产报告应查询当前 PostgreSQL。
+
 例如，查找在 ICLR 有拒稿记录、但在其他已知投稿中已有录用的论文：
 
 ```sh
-paper-atlas query --db /data/paper-index/papers.sqlite \\
+paper-atlas query --pg-service foundry-research \
   --venue ICLR --status rejected --outcome accepted
 ```
 
 `known_acceptances` 列出目前仍为录用状态的各次投稿及会议、年份、track 和决定证据，`publication_outcome` 汇总为 accepted 或 no_known_acceptance。汇总会检查所有会议和 track，不把某次拒稿覆盖到整篇论文；同一次投稿的后续撤稿也会更新该次状态并保留原决定。no_known_acceptance 只表示现有来源中没有查到录用，不能称为“从未录用”。
 
 `--stats` 中 papers 是当前规则下的论文实体数；source_records 是来源记录数；decisions 是决定观察数。一个投稿可能有多个来源和多条观察，不能相加当作论文数。
+
+已有稳定标识或原来源记录匹配时，该身份优先；同标题、同作者的另一 arXiv 存档或扩展稿不会仅因相似标题被并入。没有身份锚点时，作者佐证的精确标题只有一个候选才可匹配；多个候选会报 `Ambiguous paper identity`，整个采集文件回滚并记录失败运行。应补有证据的标识或先核查版本关系，不删除重复候选或任取第一个。一个新来源明确同时携带两个既有稳定标识时，仍可按既有规则归并并保留投稿、标签和合并日志；未归并的同标题关系继续保留为待核查候选。这些规则不证明全库去重完成。
+
+## 5. 投稿历史关系审计
+
+```sh
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas audit histories \
+  --pg-service foundry-research --pg-schema research --examples 20 \
+  --out /data/paper-index/collected/history-audit.json
+```
+
+使用一个只读一致性快照，区分论文实体、投稿和决定观察；分别统计同次投稿曾出现负面与录用观察、不同投稿共存负面与当前已知录用、不同会议之间的共存。workshop/main 是不同 track，属于同一会议时不算跨会议；会议别名也不会增加一个会议。当前已知录用使用与 `query --outcome` 相同的“每次投稿最新观察”规则，后续撤稿的该次投稿不再算当前录用，但旧录用决定仍保留。
+
+报告不凭观察时间、届次年份或行顺序断言“拒稿发生在录用之前”。样例保留真实 `decision_date`、`observed_at`、原始标签、证据 URL 和决定来源，供进一步核查；日期缺失仍保持未知。`negative` 统计包含 rejected、desk_rejected 和 withdrawn，其原始状态不互相替换。
+
+总数覆盖快照内全部投稿，样例由 `--examples` 限定为 0–100 篇；每篇最多展示 20 次投稿、每次最多 50 条决定、每条决定最多 100 个来源，并明确标记截断。样例优先保留负面/录用的不同投稿对。四项来源关系检查报告缺失关联和来源论文归属不一致，不自动修复。
+
+这些统计依赖现有论文身份关联，不证明实体合并正确，也不证明原始决定签名、公开拒稿全集、标签类型或最终全库去重已完成。SQLite 独立样本可使用 `--db`；冻结备份不能代表当前 PostgreSQL 的历史情况。

docs/www-archive.md

@@ -0,0 +1,111 @@
+# Early WWW archive collector
+
+`paper-atlas collect www-archive` is a maintained metadata adapter. It reads official
+IW3C2 archive HTML, keeps exact source hashes, and separates importable rows from
+program entries that still need review. It fetches no PDF, follows no paper-body or
+reference link, and makes no database connection.
+
+For a fresh source snapshot on the server:
+
+```sh
+paper-atlas collect www-archive \
+  --http-cache /data/paper-index/cache/www-archive \
+  --source-cache /data/paper-index/collected/www-archive-source-v2 \
+  --out /data/paper-index/collected/www-archive-normalized-v2
+```
+
+For reproducible offline replay of the already verified snapshot:
+
+```sh
+paper-atlas collect www-archive \
+  --source-cache /data/paper-index/collected/www-early-archive-source-v1 \
+  --source-cache-sha256 c9388ccef44486e0684c9ee9e8620a465cdc4574e6bc9ea9391d743e01a3312c \
+  --out /data/paper-index/collected/www-archive-normalized-v1
+```
+
+Use a new output directory for another batch. Existing snapshots and normalized
+batches are never overwritten. The manifest hash binds every source URL, byte count,
+response hash and cache filename. Plain files must remain directly within the cache;
+symlinks, escaping paths, duplicate URLs, non-200 responses and altered bytes fail.
+The command uses UTF-8 where valid and explicitly records an ISO-8859-1 fallback for
+older pages; original bytes remain authoritative for encoding corrections.
+
+## Evidence and outputs
+
+- WWW3 Spring 1995 papers use `event_key=www-1995-spring`, `track=main`. The author,
+  session and abstract indexes must have the same nonempty entry-ID set. The ID is
+  local to this exact event and proceedings section, never a DOI or OpenReview ID.
+- Posters remain `track=poster`. An abstract without a linked author-index entry
+  stays in review even when another index has a placeholder `#` link for it.
+- Paper URLs must agree between author and session indexes. A malformed whitespace
+  URL in the abstract list may use the exact event/entry-ID match only when both
+  independent indexes corroborate the same clean URL. The original href is retained.
+- Canonical names come from the explicitly labeled author index. The source lists
+  them alphabetically; this is **not** original byline order. Both the raw index names
+  and the full abstract-page byline/affiliation text remain in metadata. No person ID
+  or cross-source identity is inferred from a name. Conflicting titles or missing
+  abstracts stay in review. Topic keywords are not treated as an abstract.
+- WWW1 1994 formal presentations and WWW2 1994 mixed categories remain review
+  candidates. The first list contains uncertain business-track annotations and a
+  count discrepancy; the second contains panels, tutorials, products and other
+  activities. Neither presentation listing nor a missing full-text link establishes
+  a regular-paper acceptance or a rejection. Raw category/title/name claims are kept.
+
+The main and poster JSONL files have independent manifests with explicit event key,
+track, count and SHA256. `source_listing_count` records the full recognized abstract
+listing, while `held_entry_count` records excluded entries; neither is a ready-row
+count. Their status stays `partial`: a verified linked subset does
+not prove complete event coverage. `review-candidates.jsonl` contains nested
+candidates and reasons, with no importable decision status. It is excluded from
+`import-queue.json`. `report.json` accounts for ready/held records and every source
+used. Newlines delimit JSONL; Unicode U+2028/U+2029 inside metadata are not line breaks.
+The output is published only after all required source and scope checks pass.
+
+## Before importing
+
+This independent collector branch starts from main; it does not include or silently
+install the event-aware writer. Its output requires the reviewed integrated runtime
+from PR31 (or a main revision containing that implementation). The production event
+schema migration has been independently accepted, preserving existing edition1212
+and its 59 Fall submissions; that is not an import of the Spring proceedings.
+
+Inspect held entries and original author names, preflight every ready row against
+current PostgreSQL stable identifiers and candidate identities, and verify the
+source/year/track/event coverage before running `import records`. When a ready subset
+excludes held entries, preserve the independently known listing total in the existing
+incremental-coverage protocol: bind a freshly checked parent count and expected count,
+and set the full expected count from this manifest. A new source uses previous count
+0 and previous expected count null; an existing source requires its actual current
+values. These live parent values are not guessed by the offline collector. Do not automatically
+run this queue against an older writer. Preserve extended versions and independent
+submission/decision evidence, and keep unresolved identities held. A normalization
+report or passing tests alone do not establish production import, complete historical
+coverage or final global deduplication.
+
+Initial fixed-cache replay (64ced7a, CI1057): 38 main papers and38 posters ready;
+372 entries held (50 first-conference fields,307 mixed program entries,15 posters).
+Independent report44dfd070 verified source hashes,counts and author evidence, finding
+one existing exact-title candidate. That publication is explicitly linked to
+arXiv1811.12161 by its WWW1995/journal-reference declaration; arXiv1810.07232 is a
+different1997 invited chapter and remains separate despite the overlap note. This
+is a preparation observation, not evidence of completed production import or final
+global deduplication.
+
+## 以单篇官方页面核实待处理 poster
+
+作者索引漏链接、摘要目录使用短标题或标题存在拼写差异时，不放宽全局标题匹配。可以先取得单篇官方 HTML，人工核对 H1、署名区与原目录同一 entry URL，再提供带 SHA256 的逐条核查文件：
+
+```sh
+paper-atlas collect www-archive \
+  --source-cache /data/paper-index/collected/www-poster-reviewed-source-v2 \
+  --source-cache-sha256 8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e \
+  --reviewed-posters docs/coverage/www-1995-posters-reviewed-2026-11-09.json \
+  --reviewed-posters-sha256 <核查文件的SHA256> \
+  --out <新的输出目录>
+```
+
+[1995 年15条poster的核查输入](coverage/www-1995-posters-reviewed-2026-11-09.json) 绑定61页来源快照：原46页和补取的15页单篇HTML。程序验证输入文件及快照哈希、单篇页面哈希、完全一致的 canonical H1、原摘要目录中的同一URL/entry ID及作者证据，不通过相似标题猜身份。规范标题采用单篇H1，摘要目录标题及作者索引标题均在 metadata 中保留。
+
+`author_source=canonical_byline` 表示名字出现在H1之后、摘要/关键词或下一节标题之前的署名区域，由逐条核查明确选出，不从全文引用或机构猜作者；保留原始缩写、顺序、重音和拼写。只有 `author_source=author_index` 时使用原作者索引顺序并逐项核对；05的单篇页面写的是联系人，作者身份使用原作者索引证据，不由联系人角色推断。核查输入是审阅后的数据声明，程序的字符串/哈希检查不代替人工识别署名语义。
+
+38和05的来源没有可明确引用的摘要。保留 `abstract=null` 和逐条缺失原因，manifest准确报告 missing_abstract_count，继续标partial；正文、联系方式及PostScript链接不当作摘要，也不下载正文。缺摘要不等于拒稿或未发表。本批输出仍包含原先已验收记录，正式导入前必须只取新增external ID，检查实时覆盖父记录并按incremental模式准备，不能直接重导入全部输出。

paper_atlas/audits/coverage.py

@@ -1,45 +1,138 @@
-"""Show every target venue, including those with no imported records."""
+"""Describe imported venue coverage without implying a complete census."""
 import argparse
-from paper_atlas.config import DEFAULT_REGISTRY
+from collections import defaultdict
 import json
 from pathlib import Path
 import sqlite3
 
+from paper_atlas.config import DEFAULT_REGISTRY
+from paper_atlas.db.backend import database_arguments, open_read_connection
 from paper_atlas.db.submissions import timestamp
+from paper_atlas.querying import LATEST_DECISION
 
 
-def report(db,registry):
-    registry=json.loads(Path(registry).read_text());targets=registry['venues']
-    names=[row['acronym'].casefold() for row in targets]
-    if len(names)!=len(set(names)):raise ValueError('Duplicate target venue')
-    scope='(v.acronym=? COLLATE NOCASE OR EXISTS(SELECT 1 FROM venue_aliases a WHERE a.venue_id=v.id AND a.alias=? COLLATE NOCASE))'
-    c=sqlite3.connect(Path(db).resolve().as_uri()+'?mode=ro',uri=True);c.row_factory=sqlite3.Row
+STATUSES = ('accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown')
+
+
+def _status_counts(connection, *, latest):
+    if latest:
+        query = '''SELECT s.edition_id,COALESCE(d.status,'unknown') AS status,count(*) AS count
+            FROM submissions s LEFT JOIN decisions d ON d.submission_id=s.id AND ''' + LATEST_DECISION
+    else:
+        query = '''SELECT s.edition_id,COALESCE(d.status,'unknown') AS status,count(*) AS count
+            FROM submissions s JOIN decisions d ON d.submission_id=s.id'''
+    counts = defaultdict(dict)
+    for row in connection.execute(query + " GROUP BY s.edition_id,COALESCE(d.status,'unknown')"):
+        counts[row[0]][row[1]] = row[2]
+    return counts
+
+
+def report(db, registry, *, pg_service=None, pg_schema=None):
+    started_at = timestamp()
+    registry = json.loads(Path(registry).read_text())
+    targets = registry['venues']
+    names = [row['acronym'].casefold() for row in targets]
+    if len(names) != len(set(names)):
+        raise ValueError('Duplicate target venue')
+    scope = '''(lower(v.acronym)=lower(?) OR EXISTS(
+        SELECT 1 FROM venue_aliases a WHERE a.venue_id=v.id AND lower(a.alias)=lower(?)))'''
+    connection = open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)
+    if not pg_service:
+        connection.row_factory = sqlite3.Row
     try:
-        c.execute('BEGIN');venues=[]
+        if pg_service:
+            identity = connection.execute('SELECT current_database(),current_schema()').fetchone()
+            database, schema = identity[0], identity[1]
+        else:
+            database, schema = str(Path(db).resolve()), None
+        from paper_atlas.db.events import has_event_key
+        event = 'e.event_key' if has_event_key(connection) else "''"
+        latest = _status_counts(connection, latest=True)
+        observations = _status_counts(connection, latest=False)
+        # Aggregate the paper join once for the whole snapshot. Repeating it
+        # for each registry target caused repeated scans on the production DB.
+        edition_counts = {}
+        for row in connection.execute('''SELECT s.edition_id,count(s.id) AS submissions,
+                count(DISTINCT s.paper_id) AS papers,
+                count(DISTINCT CASE WHEN p.abstract IS NULL OR p.abstract='' THEN s.paper_id END)
+                    AS papers_without_abstract_text
+                FROM submissions s LEFT JOIN papers p ON p.id=s.paper_id GROUP BY s.edition_id'''):
+            entry = dict(row)
+            edition_counts[entry.pop('edition_id')] = entry
+        source_counts = defaultdict(list)
+        for row in connection.execute('''SELECT s.edition_id,so.name AS source,
+                count(DISTINCT sr.id) AS source_records,
+                count(DISTINCT CASE WHEN sr.abstract IS NULL OR sr.abstract='' THEN sr.id END)
+                    AS source_records_without_abstract_text
+                FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id
+                JOIN source_records sr ON sr.id=ss.source_record_id JOIN sources so ON so.id=sr.source_id
+                GROUP BY s.edition_id,so.name ORDER BY s.edition_id,so.name'''):
+            entry = dict(row)
+            source_counts[entry.pop('edition_id')].append(entry)
+        venues, covered_venue_ids = [], set()
         for target in targets:
-            acronym=target['acronym'];params=(acronym,acronym)
-            editions=[]
-            for row in c.execute('SELECT e.id,v.acronym AS canonical_venue,e.year,e.track,count(s.id) AS submissions,count(DISTINCT s.paper_id) AS papers FROM editions e JOIN venues v ON v.id=e.venue_id LEFT JOIN submissions s ON s.edition_id=e.id WHERE '+scope+' GROUP BY e.id ORDER BY e.year,e.track',params):
-                edition=dict(row)
-                edition['sources']=[dict(source) for source in c.execute('SELECT so.name,es.status,es.expected_count,es.imported_count,es.list_url,es.last_success_at,es.last_error FROM edition_sources es JOIN sources so ON so.id=es.source_id WHERE es.edition_id=? ORDER BY so.name',(row['id'],))]
+            acronym = target['acronym']
+            params = (acronym, acronym)
+            editions = []
+            for row in connection.execute('''SELECT e.id,v.id AS venue_id,v.acronym AS canonical_venue,
+                    e.year,e.track,''' + event + ''' AS event_key FROM editions e JOIN venues v ON v.id=e.venue_id
+                    WHERE ''' + scope + '''
+                    ORDER BY e.year,e.track,event_key,v.acronym,e.id''', params):
+                edition = dict(row)
+                edition.update(edition_counts.get(row['id'], {
+                    'submissions': 0, 'papers': 0, 'papers_without_abstract_text': 0}))
+                covered_venue_ids.add(edition['venue_id'])
+                edition['latest_observed_status_counts'] = {status: latest[row['id']].get(status, 0) for status in STATUSES}
+                edition['decision_observation_counts'] = {status: observations[row['id']].get(status, 0) for status in STATUSES}
+                edition['source_record_counts'] = source_counts[row['id']]
+                edition['sources'] = [dict(source) for source in connection.execute('''SELECT so.name,
+                    es.status,es.expected_count,es.imported_count,es.list_url,es.last_success_at,es.last_error
+                    FROM edition_sources es JOIN sources so ON so.id=es.source_id
+                    WHERE es.edition_id=? ORDER BY so.name,es.list_url''', (row['id'],))]
                 editions.append(edition)
-            totals=c.execute('SELECT count(s.id),count(DISTINCT s.paper_id) FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+scope,params).fetchone()
-            venues.append({'acronym':acronym,'field':target['field'],'imported_submissions':totals[0],'distinct_papers':totals[1],
-                           'history_status':'partial_history_pending_independent_audit' if totals[0] else 'no_imported_records',
-                           'discovery_sources':target.get('discovery_sources',[]),'editions':editions})
-        return {'checked_at':timestamp(),'target_venue_count':len(targets),'venues_with_imported_records':sum(bool(v['imported_submissions']) for v in venues),
-                'scope_note':registry['scope_note'],'interpretation':'All registry targets are listed. Edition source complete means that source file passed import checks; it does not prove full historical or abstract coverage. Papers can appear in multiple venues/years, so do not sum venue paper counts as a global total.',
-                'venues':venues}
-    finally:c.close()
+            totals = connection.execute('''SELECT count(s.id),count(DISTINCT s.paper_id)
+                FROM submissions s JOIN editions e ON e.id=s.edition_id
+                JOIN venues v ON v.id=e.venue_id WHERE ''' + scope, params).fetchone()
+            venues.append({'acronym': acronym, 'field': target['field'],
+                'imported_submissions': totals[0], 'distinct_papers': totals[1],
+                'history_status': 'partial_history_pending_independent_audit' if totals[0] else 'no_imported_records',
+                'discovery_sources': target.get('discovery_sources', []), 'editions': editions})
+        outside_registry = [dict(row) for row in connection.execute('''SELECT v.id AS venue_id,
+            v.acronym,count(s.id) AS submissions,count(DISTINCT s.paper_id) AS papers
+            FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
+            GROUP BY v.id,v.acronym ORDER BY v.acronym''') if row[0] not in covered_venue_ids]
+        return {'report_started_at': started_at, 'checked_at': timestamp(),
+            'engine': 'postgresql' if pg_service else 'sqlite', 'database': database, 'schema': schema,
+            'pg_service': pg_service, 'target_venue_count': len(targets),
+            'venues_with_imported_records': sum(bool(v['imported_submissions']) for v in venues),
+            'scope_note': registry['scope_note'],
+            'interpretation': 'All registry targets are listed, including zero imports. '
+                'Latest status counts use the latest observed decision for each attempt; decision observations retain history. '
+                'Source record totals are per source and edition, not per list URL; multiple list entries may describe overlapping records. '
+                'Missing abstract text means NULL or empty string; nonempty text does not prove a complete abstract. '
+                'An edition source marked complete is not a proof of complete conference history or public rejection coverage. '
+                'Zero imported rejections does not establish that no rejections are public. '
+                'Papers can appear in several venues/years; do not sum venue paper counts as a global total.',
+            'venues': venues, 'imported_venues_outside_registry': outside_registry}
+    finally:
+        connection.close()
 
 
 def main(argv=None, *, prog=None):
-    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--db',required=True)
-    parser.add_argument('--registry',default=str(DEFAULT_REGISTRY));parser.add_argument('--out')
-    args=parser.parse_args(argv);result=report(args.db,args.registry);text=json.dumps(result,ensure_ascii=False,indent=2)
+    parser = argparse.ArgumentParser(prog=prog)
+    database_arguments(parser)
+    parser.add_argument('--registry', default=str(DEFAULT_REGISTRY))
+    parser.add_argument('--out')
+    args = parser.parse_args(argv)
+    result = report(args.db, args.registry, pg_service=args.pg_service, pg_schema=args.pg_schema)
+    text = json.dumps(result, ensure_ascii=False, indent=2)
     if args.out:
-        path=Path(args.out);stage=path.with_suffix('.part');stage.write_text(text+'\n');stage.replace(path)
-    else:print(text)
+        path = Path(args.out)
+        stage = path.with_suffix('.part')
+        stage.write_text(text + '\n')
+        stage.replace(path)
+    else:
+        print(text)
 
 
 if __name__ == '__main__':

paper_atlas/audits/histories.py

@@ -0,0 +1,200 @@
+"""Audit linked submission histories without inferring event chronology."""
+import argparse
+from collections import Counter
+import json
+from pathlib import Path
+import sqlite3
+
+from paper_atlas.db.backend import database_arguments, open_read_connection
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.querying import LATEST_DECISION
+
+
+ATTEMPTS = '''SELECT s.paper_id,s.id AS submission_id,e.venue_id,v.acronym AS venue,e.year,e.track,
+    count(d.id) AS decision_observations,
+    MAX(CASE WHEN d.status IN ('rejected','desk_rejected','withdrawn') THEN 1 ELSE 0 END)
+        AS historical_negative,
+    MAX(CASE WHEN d.status='accepted' THEN 1 ELSE 0 END) AS historical_acceptance,
+    COALESCE(MAX(CASE WHEN ''' + LATEST_DECISION + ''' THEN d.status END),'unknown')
+        AS latest_observed_status
+    FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
+    LEFT JOIN decisions d ON d.submission_id=s.id
+    GROUP BY s.paper_id,s.id,e.venue_id,v.acronym,e.year,e.track
+    ORDER BY s.paper_id,e.venue_id,s.id'''
+
+
+def _paper_counts(attempts):
+    negative = sum(row['historical_negative'] for row in attempts)
+    accepted = sum(row['latest_observed_status'] == 'accepted' for row in attempts)
+    overlap = sum(row['historical_negative'] and row['latest_observed_status'] == 'accepted'
+                  for row in attempts)
+    venues = {}
+    for row in attempts:
+        counts = venues.setdefault(row['venue_id'], [0, 0])
+        counts[0] += row['historical_negative']
+        counts[1] += row['latest_observed_status'] == 'accepted'
+    return {
+        'papers_with_submissions': 1,
+        'papers_with_multiple_submissions': int(len(attempts) > 1),
+        'papers_with_historical_negative': int(negative > 0),
+        'papers_with_current_known_acceptance': int(accepted > 0),
+        'papers_with_negative_and_acceptance_in_distinct_attempts': int(negative * accepted > overlap),
+        'papers_with_negative_and_acceptance_in_distinct_venues':
+            int(negative * accepted > sum(n * a for n, a in venues.values())),
+    }
+
+
+def _example(connection, attempts):
+    # Keep a negative/accepted pair visible even when a paper has many attempts.
+    accepted = [row for row in attempts if row['latest_observed_status'] == 'accepted']
+    first = accepted[0] if accepted else None
+    other_venue = next((row for row in accepted if row['venue_id'] != first['venue_id']), None)
+    pair = None
+    for negative in attempts:
+        if negative['historical_negative'] and first:
+            candidate = first if first['venue_id'] != negative['venue_id'] else other_venue
+            if candidate:
+                pair = (negative, candidate)
+                break
+    if pair is None:
+        for negative in attempts:
+            if negative['historical_negative'] and first:
+                candidate = first if first['submission_id'] != negative['submission_id'] else (
+                    accepted[1] if len(accepted) > 1 else None)
+                if candidate:
+                    pair = (negative, candidate)
+                    break
+    selected = {}
+    for row in list(pair or ()) + attempts:
+        if len(selected) < 20:
+            selected.setdefault(row['submission_id'], dict(row))
+    for row in selected.values():
+        row['decisions'] = []
+        for decision in connection.execute('''SELECT id,status,raw_value,evidence_url,
+                decision_date,observed_at FROM decisions WHERE submission_id=?
+                ORDER BY observed_at DESC,COALESCE(decision_date,observed_at) DESC,id DESC LIMIT 50''',
+                (row['submission_id'],)):
+            entry = dict(decision)
+            sources = [dict(source) for source in connection.execute('''
+                SELECT so.name,r.external_id,ds.source_decision_id FROM decision_sources ds
+                JOIN source_records r ON r.id=ds.source_record_id JOIN sources so ON so.id=r.source_id
+                WHERE ds.decision_id=? ORDER BY so.name,r.external_id LIMIT 101''', (entry['id'],))]
+            entry['sources'] = sources[:100]
+            entry['sources_truncated'] = len(sources) > 100
+            row['decisions'].append(entry)
+        row['decisions_truncated'] = row['decision_observations'] > len(row['decisions'])
+    paper_id = attempts[0]['paper_id']
+    return {'paper_id': paper_id,
+            'title': connection.execute('SELECT title FROM papers WHERE id=?', (paper_id,)).fetchone()[0],
+            'counts': _paper_counts(attempts), 'attempt_count': len(attempts),
+            'attempts': list(selected.values()), 'attempts_truncated': len(attempts) > len(selected),
+            'chronology': 'Not inferred from observed_at, edition year, or row order.'}
+
+
+def report(db=None, *, pg_service=None, pg_schema=None, examples=20):
+    if not 0 <= examples <= 100:
+        raise ValueError('Examples must be between 0 and 100')
+    started_at = timestamp()
+    connection = open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)
+    if not pg_service:
+        connection.row_factory = sqlite3.Row
+    try:
+        if pg_service:
+            identity = connection.execute('SELECT current_database(),current_schema()').fetchone()
+            database, schema = identity[0], identity[1]
+        else:
+            database, schema = str(Path(db).resolve()), None
+        names = ['papers_with_submissions', 'papers_with_multiple_submissions',
+                 'papers_with_historical_negative', 'papers_with_current_known_acceptance',
+                 'papers_with_negative_and_acceptance_in_distinct_attempts',
+                 'papers_with_negative_and_acceptance_in_distinct_venues']
+        totals = Counter({key: 0 for key in names})
+        totals.update(submissions=0, decision_observations=0,
+                      attempts_with_historical_negative=0,
+                      attempts_with_historical_negative_and_acceptance=0)
+        latest = Counter()
+        sample_attempts = []
+
+        def consume(rows):
+            if not rows:
+                return
+            counts = _paper_counts(rows)
+            totals.update(counts)
+            if counts['papers_with_negative_and_acceptance_in_distinct_attempts'] and len(sample_attempts) < examples:
+                sample_attempts.append(rows)
+
+        current = []
+        from paper_atlas.db.events import has_event_key
+        attempts_sql = ATTEMPTS
+        if has_event_key(connection):
+            attempts_sql = attempts_sql.replace('e.year,e.track,', 'e.year,e.track,e.event_key,', 1).replace('e.year,e.track\n', 'e.year,e.track,e.event_key\n')
+        for raw in connection.execute(attempts_sql):
+            row = dict(raw)
+            if current and row['paper_id'] != current[0]['paper_id']:
+                consume(current)
+                current = []
+            current.append(row)
+            totals['submissions'] += 1
+            totals['decision_observations'] += row['decision_observations']
+            totals['attempts_with_historical_negative'] += row['historical_negative']
+            totals['attempts_with_historical_negative_and_acceptance'] += int(
+                row['historical_negative'] and row['historical_acceptance'])
+            latest[row['latest_observed_status']] += 1
+        consume(current)
+        # These are relationship checks, not a substitute for the final entity/FK audit.
+        integrity = {}
+        checks = {
+            'submissions_without_source_links': '''SELECT count(*) FROM submissions s
+                WHERE NOT EXISTS(SELECT 1 FROM submission_sources ss WHERE ss.submission_id=s.id)''',
+            'decisions_without_source_links': '''SELECT count(*) FROM decisions d
+                WHERE NOT EXISTS(SELECT 1 FROM decision_sources ds WHERE ds.decision_id=d.id)''',
+            'submission_source_paper_mismatches': '''SELECT count(*) FROM submission_sources ss
+                JOIN submissions s ON s.id=ss.submission_id JOIN source_records r ON r.id=ss.source_record_id
+                WHERE s.paper_id<>r.paper_id''',
+            'decision_source_paper_mismatches': '''SELECT count(*) FROM decision_sources ds
+                JOIN decisions d ON d.id=ds.decision_id JOIN submissions s ON s.id=d.submission_id
+                JOIN source_records r ON r.id=ds.source_record_id WHERE s.paper_id<>r.paper_id''',
+        }
+        for name, query in checks.items():
+            integrity[name] = connection.execute(query).fetchone()[0]
+        return {'report_started_at': started_at, 'checked_at': timestamp(),
+                'engine': 'postgresql' if pg_service else 'sqlite', 'database': database,
+                'schema': schema, 'pg_service': pg_service,
+                'status': 'observed_history_audit_not_final_dedup', 'counts': dict(totals),
+                'latest_attempt_status_counts': dict(sorted(latest.items())),
+                'evidence_relationship_checks': integrity,
+                'examples': [_example(connection, rows) for rows in sample_attempts],
+                'example_limit': examples,
+                'interpretation': [
+                    'Counts use existing canonical paper links, not independently verified real-world identity.',
+                    'Negative includes rejected, desk_rejected and withdrawn observations; these remain distinct decisions.',
+                    'Known acceptance follows the latest observed decision per attempt, matching query --outcome.',
+                    'Distinct attempts/venues require separate submission/venue IDs; aliases do not create new venues.',
+                    'Coexisting evidence does not prove rejection happened before acceptance. observed_at is collection time.',
+                    'No known acceptance is not proof of never accepted. Source signatures and completeness are not established here.',
+                    'Examples are bounded; counts include all attempts and observations in one read-only snapshot.',
+                    'This does not perform entity merging, final deduplication, or label namespace/type validation.',
+                ]}
+    finally:
+        connection.close()
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    database_arguments(parser)
+    parser.add_argument('--examples', type=int, default=20)
+    parser.add_argument('--out')
+    args = parser.parse_args(argv)
+    result = report(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, examples=args.examples)
+    text = json.dumps(result, ensure_ascii=False, indent=2)
+    if args.out:
+        path = Path(args.out)
+        stage = path.with_suffix('.part')
+        stage.write_text(text + '\n')
+        stage.replace(path)
+    else:
+        print(text)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/audits/public_openreview.py

@@ -39,7 +39,9 @@ def audit(db, inventory, out, *, pg_service=None, pg_schema=None):
             if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest:
                 raise ValueError('Collected manifest differs from inventory')
             count = 0
-            for line in path.read_text().splitlines():
+            # Unicode paragraph/line separators are valid inside JSON strings;
+            # split physical byte lines, not Unicode text boundaries.
+            for line in path.read_bytes().splitlines():
                 row = json.loads(line)
                 if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')):
                     raise ValueError('Record edition differs from inventory')

paper_atlas/cli.py

@@ -6,11 +6,15 @@ from paper_atlas import __version__
 
 
 COMMANDS = {
+    ('annotate',): ('paper_atlas.curation.annotations', {}, '管理独立的研究类型与自定义标签证据'),
+    ('curate','identifiers'): ('paper_atlas.curation.identifiers', {}, '按证据补充稳定标识并生成独立待导入文件'),
     ('query',): ('paper_atlas.querying', {}, '查询论文、标签和投稿历史'),
     ('coverage',): ('paper_atlas.audits.coverage', {}, '查看全部候选会议的覆盖情况'),
     ('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
     ('collect','pmlr'): ('paper_atlas.collectors.pmlr', {}, 'PMLR 官方元数据'),
     ('collect','acl'): ('paper_atlas.collectors.acl', {}, 'ACL Anthology XML'),
+    ('collect','iclr-archive'): ('paper_atlas.collectors.iclr_archive', {}, '核对ICLR 2014–2016官方历史录用档案'),
+    ('collect','rss-proceedings'): ('paper_atlas.collectors.rss', {}, '核对RSS 2025官方论文摘要与演示证据'),
     ('collect','iclr-program'): ('paper_atlas.collectors.iclr', {}, 'ICLR 官方公开程序'),
     ('collect','iclr-snapshot'): ('paper_atlas.collectors.openreview', {}, 'ICLR 公开投稿与决定快照'),
     ('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
@@ -21,10 +25,15 @@ COMMANDS = {
     ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
     ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
     ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
+    ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
+    ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
     ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
     ('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'),
     ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
     ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
+    ('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
+    ('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
+    ('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
     ('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
     ('maintain','backfill'): ('paper_atlas.ingestion.backfill', {}, '回填既有投稿的来源身份'),
     ('maintain','arxiv-tags'): ('paper_atlas.ingestion.arxiv_tags', {}, '恢复逐来源记录的 arXiv 类别证据'),
@@ -35,6 +44,7 @@ COMMANDS = {
     ('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
     ('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
     ('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
+    ('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
 }
 
 
@@ -47,7 +57,7 @@ def main(argv=None):
             leaf=commands.add_parser(route[0],help=help_text,add_help=False)
         else:
             if route[0] not in groups:
-                group=commands.add_parser(route[0],help={'collect':'采集来源元数据','catalogue':'盘点历史目录','import':'导入数据库','maintain':'维护数据库','audit':'核验数据'}[route[0]])
+                group=commands.add_parser(route[0],help={'collect':'采集来源元数据','catalogue':'盘点历史目录','import':'导入数据库','maintain':'维护数据库','audit':'核验数据','curate':'按证据整理元数据'}[route[0]])
                 groups[route[0]]=group.add_subparsers(dest='operation',required=True)
             leaf=groups[route[0]].add_parser(route[1],help=help_text,add_help=False)
         leaf.set_defaults(handler=(route,module,kwargs))

paper_atlas/collectors/aamas.py

@@ -0,0 +1,210 @@
+"""Official AAMAS proceedings metadata, with publication format separate from track."""
+import argparse
+from collections import Counter, defaultdict
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import re
+import tempfile
+from urllib.parse import urljoin, urlsplit
+
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.io.html import Document, Node, clean
+from paper_atlas.io.http import fetch
+
+SOURCE = 'aamas_proceedings'
+ROOT = 'https://www.ifaamas.org/Proceedings/aamas2012/'
+# Counts independently checked against the frozen publisher sections. This
+# adapter deliberately refuses other editions/layouts until they are reviewed.
+SECTIONS = {
+    'fullpapers': ('Full Papers', 'full_paper', 'proceedings_unspecified', 136),
+    'extabstracts': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 146),
+    'demos': ('Demos', 'demo', 'demo', 28),
+    'doctorals': ('Doctorial Consortium Abstracts', 'doctoral_abstract', 'doctoral', 24),
+}
+
+
+def classes(node):
+    return set(node.attrs.get('class', '').split())
+
+
+def text_without(node, omitted):
+    if node.tag in ('script', 'style') or classes(node) & omitted:
+        return ''
+    parts = [text_without(child, omitted) if isinstance(child, Node) else child
+             for child in node.children]
+    value = ''.join(parts)
+    return ' '+value+' ' if node.tag in ('p', 'div', 'section', 'li', 'br') else value
+
+
+def one(nodes, description):
+    if len(nodes) != 1:
+        raise ValueError('Expected one '+description)
+    return nodes[0]
+
+
+def parse_section(html, section, observed_at, evidence):
+    heading, publication_format, track, _ = SECTIONS[section]
+    document = Document(html)
+    titles = [clean(n.text()) for n in document.root.walk() if n.tag == 'title']
+    if titles != [heading+' - AAMAS 2012']:
+        raise ValueError('Unexpected proceedings edition or section title')
+    headings = [clean(n.text()) for n in document.root.walk() if n.tag == 'h1']
+    if headings != [heading]:
+        raise ValueError('Unexpected proceedings section heading')
+    url = ROOT+'resources/'+section+'.html'
+    records = []
+    seen = set()
+    session = None
+    for node in document.root.walk():
+        if node.tag == 'h2' and 'section' in classes(node):
+            session = {'id': node.attrs.get('id'), 'title': clean(node.text())}
+        if node.tag != 'div' or 'paper' not in classes(node):
+            continue
+        paper_id = node.attrs.get('id', '')
+        if not re.fullmatch(r'[A-Za-z0-9]+_[0-9]+', paper_id) or paper_id in seen:
+            raise ValueError('Missing or repeated publisher paper ID')
+        seen.add(paper_id)
+        title_node = one([n for n in node.walk() if n.tag == 'span' and 'title' in classes(n)], 'paper title')
+        link = one([n for n in title_node.children if isinstance(n, Node) and n.tag == 'a'], 'paper link')
+        paper_url = urljoin(url, link.attrs.get('href', ''))
+        if paper_url != ROOT+'papers/'+paper_id+'.pdf':
+            raise ValueError('Paper link differs from publisher ID or official host')
+        title = clean(text_without(link, {'abstract'}))
+        abstract_nodes = [n for n in link.walk() if n.tag == 'span' and 'abstract' in classes(n)]
+        if len(abstract_nodes) > 1:
+            raise ValueError('Multiple abstracts for one paper')
+        abstract = (clean(abstract_nodes[0].text()) or None) if abstract_nodes else None
+        author_node = one([n for n in node.walk() if n.tag == 'span' and 'authors' in classes(n)], 'author list')
+        author_display = clean(text_without(author_node, {'amount'}))
+        authors = [clean(name) for name in author_display.split(',')]
+        if not title or not author_display or any(not name for name in authors):
+            raise ValueError('Empty paper title or author')
+        tags = [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]
+        if session:
+            tags.append({'namespace': 'aamas_proceedings:session', 'value': session['title']})
+        records.append({
+            'source': SOURCE, 'external_id': '2012/'+paper_id,
+            'url': url+'#'+paper_id, 'title': title, 'abstract': abstract,
+            'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
+            'year': 2012, 'track': track, 'status': 'accepted',
+            'observed_at': observed_at,
+            'raw_decision': 'listed in official AAMAS 2012 proceedings '+heading,
+            'metadata': {'publisher_paper_id': paper_id, 'paper_url': paper_url,
+                         'section_url': url, 'section_heading': heading,
+                         'publication_format': publication_format, 'session': session,
+                         'authors_display': author_display, 'source_evidence': evidence,
+                         'publication_evidence': 'official published proceedings entry; not an original decision note',
+                         'track_scope_note': ('Full papers and extended abstracts include general and special submission tracks; '
+                                              'session topics do not establish an individual submission track.'
+                                              if track == 'proceedings_unspecified' else 'Explicit publisher section.'),
+                         'decision_date_precision': 'not_supplied'},
+            'tags': tags})
+    if not records:
+        raise ValueError('No publisher paper entries found')
+    return records
+
+
+def read_pages(cache=None, snapshots=None):
+    if bool(cache) == bool(snapshots):
+        raise ValueError('Choose exactly one HTTP cache or frozen snapshot directory')
+    urls = {'introduction': ROOT, **{key: ROOT+'resources/'+key+'.html' for key in SECTIONS}}
+    pages = {}
+    receipts = []
+    frozen = json.loads((Path(snapshots)/'receipts.json').read_text())['receipts'] if snapshots else []
+    for key, url in urls.items():
+        if snapshots:
+            receipt = one([r for r in frozen if r['url'] == url], 'frozen response receipt')
+            filename = Path(receipt['file'])
+            if filename.is_absolute() or '..' in filename.parts or receipt.get('status') != 200:
+                raise ValueError('Unsafe or unsuccessful frozen response')
+            data = (Path(snapshots)/filename).read_bytes()
+            if len(data) > 8*1024**2 or hashlib.sha256(data).hexdigest() != receipt['sha256'] or len(data) != receipt['bytes']:
+                raise ValueError('Frozen publisher response checksum or size mismatch')
+            observed = receipt['finished_at']
+        else:
+            data = fetch(url, cache, 8*1024**2, delay=1)
+            receipt = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_text())
+            observed = receipt['retrieved_at']
+        evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
+                    'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html', 'observed_at': observed,
+                    'http_status': 200, 'response_receipt': receipt}
+        pages[key] = (data, evidence)
+        receipts.append(evidence)
+    return pages, receipts
+
+
+def _collect(year, out, *, cache=None, snapshots=None):
+    if year != 2012:
+        raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported')
+    pages, receipts = read_pages(cache, snapshots)
+    intro = Document(pages['introduction'][0].decode('utf-8'))
+    if [clean(n.text()) for n in intro.root.walk() if n.tag == 'title'] != ['AAMAS 2012']:
+        raise ValueError('Unexpected proceedings introduction')
+    records = []
+    for section, (_, _, _, expected) in SECTIONS.items():
+        data, evidence = pages[section]
+        group = parse_section(data.decode('utf-8'), section, evidence['observed_at'], evidence)
+        if len(group) != expected:
+            raise ValueError('Publisher section count differs from reviewed inventory: '+section)
+        records.extend(group)
+    if len({r['external_id'] for r in records}) != len(records):
+        raise ValueError('Paper IDs collide across publisher sections')
+    grouped = defaultdict(list)
+    for record in records:
+        grouped[record['track']].append(record)
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    files = []
+    with tempfile.TemporaryDirectory(prefix='.aamas-', dir=out) as temporary:
+        stage = Path(temporary)
+        (stage/'evidence').mkdir()
+        for data, evidence in pages.values():
+            (stage/evidence['file']).write_bytes(data)
+        for track, group in sorted(grouped.items()):
+            path = stage/f'aamas-proceedings-2012-{track}.jsonl'
+            path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
+            manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'AAMAS', 'year': 2012, 'track': track,
+                        'count': len(group), 'file': str(out/path.name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
+                        'list_url': ROOT, 'status': 'partial', 'missing_abstract_count': sum(not r['abstract'] for r in group),
+                        'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
+                        'note': 'Publisher sections verified; individual full/extended submission track, decision history and independent identities still need review.'}
+            path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
+            files.append(manifest)
+        result = {'source': SOURCE, 'year': year, 'finished_at': timestamp(), 'status': 'collected_pending_identity_review',
+                  'records': len(records), 'files': files, 'source_receipts': receipts,
+                  'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in records)),
+                  'database_written': False, 'pdfs_downloaded': False,
+                  'note': 'Aggregate submission/withdrawal totals do not identify individual rejected or withdrawn papers.'}
+        (stage/'aamas-proceedings-2012.json').write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
+        # Validated files first; write the completed collection report last.
+        (out/'evidence').mkdir(exist_ok=True)
+        for path in (stage/'evidence').iterdir():
+            path.replace(out/'evidence'/path.name)
+        for path in stage.iterdir():
+            if path.is_file() and path.name != 'aamas-proceedings-2012.json':
+                path.replace(out/path.name)
+        (stage/'aamas-proceedings-2012.json').replace(out/'aamas-proceedings-2012.json')
+    return result
+
+
+def collect(year, out, *, cache=None, snapshots=None):
+    if year != 2012:
+        raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported')
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    with (out/'aamas.collect.lock').open('a') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        return _collect(year, out, cache=cache, snapshots=snapshots)
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--year', type=int, choices=[2012], required=True)
+    parser.add_argument('--out', required=True)
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument('--cache')
+    source.add_argument('--snapshots')
+    args = parser.parse_args(argv)
+    print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))

paper_atlas/collectors/dblp.py

@@ -20,7 +20,7 @@ DTD_MD5='96e4283e71886dec2173718957293648'
 DTD_NAMES={'dblp.dtd','dblp-2023-06-28.dtd'}
 RELEASE='https://doi.org/10.4230/dblp.xml.2026-11-01'
 # Broad discovery candidates, including historical names; not a venue mapping.
-FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
+FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps ecp ewsp kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
 
 
 def records(archive,dtd_bytes):

paper_atlas/collectors/dblp_predecessors.py

@@ -0,0 +1,45 @@
+"""Exact predecessor volumes, with original names and independent family histories."""
+ECML_HISTORY = 'https://ecmlpkdd2011.org/pastConferences.php'
+ICAPS_HISTORY = 'https://www.icaps-conference.org/conference-series/'
+
+# family/year: booktitle, original series, target family, publication year, full title
+VOLUMES = {
+    ('ecml', 1987): ('EWSL', 'EWSL', 'ECML-PKDD', 1987,
+        'Progress in Machine Learning - Proceedings of EWSL 87: 2nd European Working Session on Learning, Bled, Yugoslavia, May 1987'),
+    ('ecml', 1988): ('EWSL', 'EWSL', 'ECML-PKDD', 1988,
+        'Proceedings of the Third European Working Session on Learning, EWSL 1988, Turing Institute, Glasgow, UK, October 3-5, 1988'),
+    ('ecml', 1989): ('EWSL', 'EWSL', 'ECML-PKDD', 1989,
+        'Proceedings of the 4th European Working Session on Learning, Montpellier, France, 4-6 December 1989'),
+    ('ecml', 1991): ('EWSL', 'EWSL', 'ECML-PKDD', 1991,
+        'Machine Learning - EWSL-91, European Working Session on Learning, Porto, Portugal, March 6-8, 1991, Proceedings'),
+    ('aips', 1990): ('Expert Planning Systems', 'EPS', 'ICAPS', 1990,
+        'Proceedings of the First International Conference on Expert Planning Systems, Brighton, UK, June 27-29, 1990'),
+    ('ecp', 1997): ('ECP', 'ECP', 'ICAPS', 1997,
+        "Recent Advances in AI Planning, 4th European Conference on Planning, ECP'97, Toulouse, France, September 24-26, 1997, Proceedings"),
+    ('ecp', 1999): ('ECP', 'ECP', 'ICAPS', 2000,
+        "Recent Advances in AI Planning, 5th European Conference on Planning, ECP'99, Durham, UK, September 8-10, 1999, Proceedings"),
+    ('ewsp', 1991): ('EWSP', 'EWSP', 'ICAPS', 1991,
+        "European Workshop on Planning, EWSP '91, Sankt Augustin, Germany, March 18-19, 1991, Proceedings"),
+}
+
+
+def predecessor_scope(family, base, suffix, volume, year, title):
+    definition = VOLUMES.get((family, year))
+    if definition is None:
+        return None
+    book, original, target, publication_year, expected_title = definition
+    if (suffix or base != book or volume['key'] != f'conf/{family}/{year}'
+            or title != expected_title or volume['fields'].get('year') != [str(publication_year)]):
+        return None
+    return {'original_series': original, 'target_family': target,
+            'history_url': ECML_HISTORY if target == 'ECML-PKDD' else ICAPS_HISTORY,
+            'proceedings_key': volume['key'], 'proceedings_title': title,
+            'event_year': year, 'proceedings_year_fields': list(volume['fields']['year'])}
+
+
+def missing_candidate_families(targets, families, through_year):
+    # These two families were absent from the earlier narrowed candidate JSONL,
+    # although their proceedings and volume_counts exist in the full catalogue.
+    required = {family for (family, year), definition in VOLUMES.items()
+                if family in ('ecp', 'ewsp') and definition[2] in targets and year <= through_year}
+    return sorted(family for family in required if not families.get(family))

paper_atlas/collectors/dblp_records.py

@@ -15,6 +15,7 @@ from urllib.parse import parse_qs,unquote,urlsplit
 from paper_atlas.config import DEFAULT_REGISTRY
 from paper_atlas.collectors.dblp import XML_SHA,DTD_MD5,RELEASE,records
 from paper_atlas.collectors.dblp_scope import scope
+from paper_atlas.collectors.dblp_predecessors import missing_candidate_families
 from paper_atlas.db.submissions import timestamp
 from paper_atlas.io.snapshots import digest_file
 from paper_atlas.io.publication_ids import mlsys_id
@@ -88,8 +89,14 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026)
               'scope_rule':edition['rule'],'author_names_raw':fields.get('author',[]),'publtype':row.get('publtype'),
               'bibliography':{key:value for key,value in fields.items() if key in ('booktitle','year','ee','url','pages','volume','number','publisher','series','isbn')},
               'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'}
+    if edition.get('historical_predecessor_evidence'):
+        metadata['historical_predecessor_evidence'] = edition['historical_predecessor_evidence']
+    if edition.get('joint_edition_evidence'):
+        metadata['joint_edition_evidence'] = edition['joint_edition_evidence']
     if edition.get('track_scope_note'):
         metadata['track_scope_note']=edition['track_scope_note']
+    if edition.get('track_scope_evidence_url'):
+        metadata['track_scope_evidence_url']=edition['track_scope_evidence_url']
     flagged=bool(re.search(r'\b(?:withdrawn|retracted)\b',row.get('publtype') or '',re.I))
     result={'source':'dblp','external_id':row['key'],'url':'https://dblp.org/rec/'+row['key']+'.html',
             'title':title,'abstract':None,'authors':authors,'identifiers':own,
@@ -120,8 +127,9 @@ def collect(catalogue,out,venues=None,candidates=None,archive=None,dtd=None,incl
         families = manifest.get('candidate_family_counts', {})
         missing = [venue for venue, family in [('MM', 'mm'), ('MLSys', 'mlsys')]
                    if venue in targets and not families.get(family)]
+        missing += ['ICAPS/'+family for family in missing_candidate_families(targets, families, through_year)]
         if missing:
-            raise ValueError('Candidate file predates requested families ' + ', '.join(missing) + '; use the original --archive')
+            raise ValueError('Candidate file omits requested families ' + ', '.join(missing) + '; use the original --archive')
     out=Path(out);out.mkdir(parents=True,exist_ok=True)
     with (out/'dblp.normalize.lock').open('a') as lock:
         fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)

paper_atlas/collectors/dblp_scope.py

@@ -1,6 +1,8 @@
 """Explicit bibliographic aliases; a directory family alone is not venue evidence."""
 import re
 
+from paper_atlas.collectors.dblp_predecessors import predecessor_scope
+
 
 # Ordered long/joint names precede their shorter forms.
 BOOKS = {
@@ -27,7 +29,8 @@ BOOKS = {
     'pkdd':[('ECML/PKDD','ECML-PKDD',[]),('ECML PKDD','ECML-PKDD',[]),('PKDD','PKDD',['ECML-PKDD'])],
     'icra':[('ICRA','ICRA',[])], 'iros':[('IROS','IROS',[])],
     'rss':[('Robotics: Science and Systems','RSS',[])], 'corl':[('CoRL','CoRL',[])],
-    'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])],
+    'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS']),('Expert Planning Systems','EPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])],
+    'ecp':[('ECP','ECP',['ICAPS'])], 'ewsp':[('EWSP','EWSP',['ICAPS'])],
     'kr':[('KR','KR',[])], 'atal':[('AAMAS','AAMAS',[])],
     'icassp':[('ICASSP','ICASSP',[])],
     'interspeech':[('INTERSPEECH','INTERSPEECH',[]),('EUROSPEECH','EUROSPEECH',['INTERSPEECH']),('ICSLP','ICSLP',['INTERSPEECH']),('ECST','ECST',['INTERSPEECH'])],
@@ -38,6 +41,66 @@ BOOKS = {
 }
 
 
+# These are the historical primary series themselves, not co-located workshops.
+# Exact proceedings keys, original names, ordinals and event years must agree.
+HISTORICAL_WORKSHOP_TITLES = {
+    **{('aistats', year): prefix + ordinal + ' International Workshop on Artificial Intelligence and Statistics'
+       for year, ordinal, prefix in [
+           (1995, 'Fifth', 'Learning from Data - '),
+           (1997, 'Sixth', 'Proceedings of the '),
+           (1999, 'Seventh', 'Proceedings of the '),
+           (2001, 'Eighth', 'Proceedings of the '),
+           (2003, 'Ninth', 'Proceedings of the '),
+           (2005, 'Tenth', 'Proceedings of the ')]},
+    **{('colt', year): 'Proceedings of the ' + ordinal + ' Annual Workshop on Computational Learning Theory'
+       for year, ordinal in [(1988, 'First'), (1989, 'Second'), (1990, 'Third'), (1991, 'Fourth')]},
+}
+
+
+# Exact pinned volume titles corroborated by official series histories. The
+# original title (including Workshop) remains in source metadata. In particular
+# ML91 has no four-digit year in its title; its exact key/title and bibliographic
+# year are checked independently, rather than guessing a title year.
+HISTORICAL_PRIMARY_EVIDENCE = {
+    'icml': ('ML', 'https://icml.cc/Conferences/2008/past_icmls.shtml.html'),
+    'iros': ('IROS', 'https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/'),
+}
+HISTORICAL_PRIMARY_TITLES = {
+    ('icml', 1988): 'Machine Learning, Proceedings of the Fifth International Conference on Machine Learning, Ann Arbor, Michigan, USA, June 12-14, 1988',
+    ('icml', 1989): 'Proceedings of the Sixth International Workshop on Machine Learning (ML 1989), Cornell University, Ithaca, New York, USA, June 26-27, 1989',
+    ('icml', 1990): 'Machine Learning, Proceedings of the Seventh International Conference on Machine Learning, Austin, Texas, USA, June 21-23, 1990',
+    ('icml', 1991): 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA',
+    ('icml', 1992): 'Proceedings of the Ninth International Workshop on Machine Learning (ML 1992), Aberdeen, Scotland, UK, July 1-3, 1992',
+    ('iros', 1988): "IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988",
+    ('iros', 1989): "IEEE International Workshop on Intelligent Robots and Systems '89, Proceedings. IROS 1989, September 4-6, 1989, Tsukuba, Japan",
+    ('iros', 1990): "IEEE International Workshop on Intelligent Robots and Systems '90, Towards a New Frontier of Applications, Proceedings. IROS 1990, July 3-6, 1990, Ibaraki, Japan",
+}
+
+
+def historical_track(family, base, suffix, volume_key, year, title):
+    primary = HISTORICAL_PRIMARY_EVIDENCE.get(family)
+    if (primary and not suffix and base.casefold() == primary[0].casefold()
+            and volume_key == f'conf/{family}/{year}'
+            and title == HISTORICAL_PRIMARY_TITLES.get((family, year))):
+        return ('main',
+                'Historical primary series; exact proceedings key, complete title and bibliographic/event year corroborated by official series history. Workshop in this original title is not an affiliated track.',
+                primary[1])
+    if suffix or base.casefold() != family or volume_key != f'conf/{family}/{year}':
+        return None
+    prefix = HISTORICAL_WORKSHOP_TITLES.get((family, year))
+    if (prefix and title.casefold().startswith(prefix.casefold() + ',')
+            and re.search(r'\b' + str(year) + r'\b', title)):
+        url = 'https://aistats.org/other.html' if family == 'aistats' else 'https://www.learningtheory.org/'
+        return ('main', 'Historical primary series named Workshop; exact proceedings key, ordinal, title and event year verified.', url)
+    if (family == 'colt' and year == 2003
+            and title.startswith('Computational Learning Theory and Kernel Machines, 16th Annual Conference on Computational Learning Theory and 7th Kernel Workshop, COLT/Kernel 2003,')
+            and title.endswith('Proceedings')):
+        return ('proceedings_unspecified',
+                'Combined COLT 2003 and Kernel Workshop volume; per-paper section absent from DBLP XML. Do not infer a main or workshop track.',
+                'https://doi.org/10.1007/b12006')
+    return None
+
+
 def scope(row,volume,through_year=2026):
     fields=row['fields'];books=fields.get('booktitle',[])
     if len(books)!=1:return None,'ambiguous_booktitle'
@@ -75,7 +138,13 @@ def scope(row,volume,through_year=2026):
     if len(bibliographic)!=1 or not re.fullmatch(r'\d{4}',bibliographic[0]):return None,'ambiguous_bibliographic_year'
     if int(bibliographic[0])!=year and not re.search(r'\b'+str(year)+r'\b',title):return None,'unverified_year_difference'
     text=(suffix+' '+title).casefold();track='main'
-    if 'findings' in text:track='findings'
+    predecessor = predecessor_scope(family, base, suffix, volume, year, title)
+    if (family in ('ecp', 'ewsp') or (family == 'aips' and base == 'Expert Planning Systems')) and not predecessor:
+        return None, 'unverified_historical_predecessor'
+    historical = (('main', 'Historical predecessor; exact volume, full title, event and publication years verified; original series retained.', predecessor['history_url'])
+                  if predecessor else historical_track(family, base, suffix, volume['key'], year, title))
+    if historical:track=historical[0]
+    elif 'findings' in text:track='findings'
     elif 'shared task' in text:track='shared_task'
     elif 'student research' in text or 'workshop' in text or 'wokshop' in text or base in ('ICCVW','EWSL'):track='workshop'
     elif ('demo track' in text and ('industrial track' in text or 'applied data science track' in text)):track='proceedings_unspecified'
@@ -96,8 +165,20 @@ def scope(row,volume,through_year=2026):
     elif suffix.casefold() in ('poster','posters') and venue not in ('ICLR','ACL','EMNLP','NAACL','EACL','COLING','CoNLL'):track='poster'
     if venue=='AAAI+IAAI':track='proceedings_unspecified'
     if venue=='AAMAS' and year<2002:return None,'predecessor_not_aamas'
+    joint_evidence = None
+    if (family == 'ijcai' and base == 'IJCAI' and track == 'main'
+            and year in (2018, 2022) and volume['key'] == 'conf/ijcai/' + str(year)):
+        venue, aliases = 'ECAI+IJCAI', ['ECAI', 'IJCAI']
+        joint_evidence = 'https://www.eurai.org/ecai/history'
     result = {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]),
               'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis}
+    if predecessor:
+        result['historical_predecessor_evidence'] = predecessor
+    if joint_evidence:
+        result['joint_edition_evidence'] = joint_evidence
+    if historical:
+        result['track_scope_note'] = historical[1]
+        result['track_scope_evidence_url'] = historical[2]
     if venue=='MM' and track=='main':
         # A main MM volume also includes doctoral, tutorials, companion papers
         # and other sections. DBLP XML omits the per-paper TOC section.

paper_atlas/collectors/iclr_archive.py

@@ -0,0 +1,225 @@
+"""Frozen official ICLR 2014–2016 lists; presentation evidence is not a track."""
+import argparse
+from collections import Counter, defaultdict
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import re
+import tempfile
+from urllib.parse import urlsplit
+
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.io.html import Document, Node, clean
+from paper_atlas.io.http import fetch
+
+SOURCE = 'iclr_archive'
+PAGES = {
+    2014: ('https://iclr.cc/archive/2014/conference-proceedings/', {'main': 35}),
+    2015: ('https://iclr.cc/archive/www/doku.php%3Fid=iclr2015:accepted-main.html', {'main': 31, 'workshop': 74}),
+    2016: ('https://iclr.cc/archive/www/doku.php%3Fid=iclr2016:accepted-main.html', {'main': 80}),
+}
+SECTIONS = {
+    2015: {'Main Conference - Oral Presentations': ('main', 'oral'),
+           'Main Conference - Poster Presentations': ('main', 'poster'),
+           'Workshop Papers': ('workshop', 'workshop')},
+    2016: {'Oral Presentations': ('main', 'oral'), 'Poster Presentations': ('main', 'poster')},
+}
+
+
+def arxiv_link(url):
+    parsed = urlsplit(url)
+    if parsed.hostname != 'arxiv.org':
+        return None
+    match = re.fullmatch(r'/(?:abs|pdf)/(\d{4}\.\d{4,5})(v[1-9]\d*)?(?:\.pdf)?', parsed.path)
+    if (parsed.scheme not in ('http', 'https') or parsed.netloc != 'arxiv.org'
+            or parsed.query or parsed.fragment or not match):
+        raise ValueError('Unexpected official arXiv link')
+    return match[1], match[2] or None
+
+
+def without_links(node):
+    if node.tag in ('a', 'script', 'style'):
+        return ''
+    value = ''.join(without_links(n) if isinstance(n, Node) else n for n in node.children)
+    return ' '+value+' ' if node.tag in ('div', 'li', 'p', 'br') else value
+
+
+def authors(display, year):
+    parts = display.rstrip(';').split(';') if year == 2014 else re.split(r',\s*(?:and\s+)?|\s+and\s+', display)
+    names = [clean(p) for p in parts]
+    if not names or any(not name for name in names):
+        raise ValueError('Missing author or unsupported author delimiter')
+    return names
+
+
+def parse_page(html, year, evidence):
+    if year not in PAGES:
+        raise ValueError('Unreviewed ICLR archive year')
+    document = Document(html)
+    nodes = list(document.root.walk())
+    headings = [clean(n.text()) for n in nodes if n.tag in ('h1', 'h2', 'h3')]
+    if year == 2014:
+        if 'Conference Proceedings' not in headings or 'conference papers accepted' not in clean(document.root.text()):
+            raise ValueError('Unexpected ICLR 2014 accepted list')
+    elif (set(SECTIONS[year]) - set(headings)
+          or (year == 2016 and 'Accepted Papers (Conference Track)' not in headings)):
+        raise ValueError('Unexpected ICLR accepted sections')
+    entries = []
+    section = None
+    for index, node in enumerate(nodes):
+        if node.tag in ('h1', 'h2', 'h3'):
+            section = clean(node.text())
+        if year == 2014:
+            if node.tag != 'a' or not arxiv_link(node.attrs.get('href', '')):
+                continue
+            link = node
+            # This archived Google Sites page has an entry outside a paragraph.
+            # Authors can span several differently formatted italic spans.
+            # Read their whole paragraph, never only the first styled fragment.
+            display = None
+            for following in nodes[index+sum(1 for _ in link.walk()):]:
+                if following.tag == 'a' and arxiv_link(following.attrs.get('href', '')):
+                    break
+                if following.tag in ('i', 'em') or (following.tag == 'p' and any(
+                        n.tag in ('i', 'em') or 'font-style:italic' in n.attrs.get('style', '').replace(' ', '')
+                        for n in following.walk())):
+                    text = clean(following.text())
+                    if text:
+                        display = text
+                        break
+            if not display:
+                raise ValueError('Missing explicit ICLR 2014 author span')
+            track, presentation = 'main', 'conference'
+        else:
+            if node.tag != 'li':
+                continue
+            links = [n for n in node.walk() if n.tag == 'a' and arxiv_link(n.attrs.get('href', ''))]
+            if not links:
+                continue
+            if len(links) != 1 or section not in SECTIONS[year]:
+                raise ValueError('Ambiguous paper link or unknown accepted section')
+            link, = links
+            display = clean(without_links(node)).lstrip(',').strip()
+            track, presentation = SECTIONS[year][section]
+        arxiv, version = arxiv_link(link.attrs['href'])
+        title = clean(link.text())
+        if not title:
+            raise ValueError('Empty official title')
+        entries.append({'arxiv': arxiv, 'version': version, 'title': title,
+                        'authors': authors(display, year), 'authors_display': display,
+                        'paper_url': link.attrs['href'], 'track': track,
+                        'section': section, 'presentation': presentation})
+    # Check every arXiv occurrence, including links outside the expected layout.
+    occurrences = sum(n.tag == 'a' and bool(arxiv_link(n.attrs.get('href', ''))) for n in nodes)
+    if len(entries) != occurrences or not entries:
+        raise ValueError('Some official paper links were not parsed')
+    grouped = defaultdict(list)
+    for entry in entries:
+        grouped[(entry['track'], entry['arxiv'])].append(entry)
+    result = []
+    for (track, arxiv), variants in grouped.items():
+        first = variants[0]
+        if any((v['title'], v['authors']) != (first['title'], first['authors']) for v in variants):
+            raise ValueError('Repeated presentation has conflicting title or authors; review required')
+        presentations = sorted({v['presentation'] for v in variants})
+        result.append({'source': SOURCE, 'external_id': f'{year}/{track}/{arxiv}',
+                       'url': PAGES[year][0], 'title': first['title'], 'abstract': None,
+                       'authors': first['authors'], 'identifiers': {'arxiv': arxiv},
+                       'venue': 'ICLR', 'venue_aliases': [], 'year': year, 'track': track,
+                       'status': 'accepted', 'observed_at': evidence['observed_at'],
+                       'raw_decision': 'listed in official ICLR accepted '+track+' list',
+                       'metadata': {'source_evidence': evidence, 'source_variants': variants,
+                                    'presentations': presentations, 'decision_date_precision': 'not_supplied',
+                                    'identity_note': 'Official arXiv link, not title matching; validate canonical identity before import.',
+                                    'decision_evidence': 'Archived accepted list, not an original decision note.'},
+                       'tags': [{'namespace': SOURCE+':presentation', 'value': value} for value in presentations]})
+    return result
+
+
+def read_page(year, *, cache=None, snapshots=None):
+    if bool(cache) == bool(snapshots):
+        raise ValueError('Choose exactly one cache or frozen snapshot directory')
+    url = PAGES[year][0]
+    if snapshots:
+        root = Path(snapshots).resolve()
+        receipts = json.loads((root/'receipts.json').read_bytes())['receipts']
+        matches = [r for r in receipts if r['url'] == url]
+        if len(matches) != 1:
+            raise ValueError('Expected one exact official response receipt')
+        receipt, = matches
+        filename = Path(receipt['file'])
+        path = root/filename
+        if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
+                or receipt['status'] != 200):
+            raise ValueError('Unsafe or unsuccessful archive snapshot')
+        data = path.read_bytes()
+        if (len(data) > 8*1024**2 or len(data) != receipt['bytes']
+                or hashlib.sha256(data).hexdigest() != receipt['sha256']):
+            raise ValueError('Archive snapshot checksum or size differs')
+        observed = timestamp(receipt['finished_at'])
+    else:
+        data = fetch(url, cache, 8*1024**2, delay=1)
+        receipt = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_bytes())
+        observed = timestamp(receipt['retrieved_at'])
+    evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
+                'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
+                'observed_at': observed, 'http_status': 200, 'response_receipt': receipt}
+    return data, evidence
+
+
+def collect(year, out, *, cache=None, snapshots=None):
+    if year not in PAGES:
+        raise ValueError('Only independently reviewed ICLR archive years are supported')
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    with (out/'iclr-archive.collect.lock').open('a') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        report = out/f'iclr-archive-{year}.json'
+        if report.exists():
+            raise FileExistsError('Existing completed archive output must be inspected, not overwritten')
+        data, evidence = read_page(year, cache=cache, snapshots=snapshots)
+        records = parse_page(data.decode('utf-8'), year, evidence)
+        counts = Counter(r['track'] for r in records)
+        if dict(counts) != PAGES[year][1]:
+            raise ValueError('Distinct archive entries differ from reviewed inventory')
+        files = []
+        with tempfile.TemporaryDirectory(prefix='.iclr-archive-', dir=out) as temporary:
+            stage = Path(temporary)
+            (stage/'evidence').mkdir()
+            (stage/evidence['file']).write_bytes(data)
+            for track in sorted(counts):
+                group = [r for r in records if r['track'] == track]
+                path = stage/f'iclr-archive-{year}-{track}.jsonl'
+                path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
+                manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year,
+                            'track': track, 'count': len(group), 'file': str(out/path.name),
+                            'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
+                            'list_url': PAGES[year][0], 'status': 'partial', 'missing_abstract_count': len(group)}
+                path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
+                files.append(manifest)
+            result = {'source': SOURCE, 'year': year, 'records': len(records), 'files': files,
+                      'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
+                      'status': 'collected_pending_identity_review',
+                      'note': 'Accepted list only. Oral/poster are presentation evidence; absent papers are not rejected. '
+                              'ICLR2014 workshop and other unlisted tracks remain outside these reviewed pages.'}
+            (stage/report.name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
+            (out/'evidence').mkdir(exist_ok=True)
+            for path in (stage/'evidence').iterdir():
+                path.replace(out/'evidence'/path.name)
+            for path in stage.iterdir():
+                if path.is_file() and path.name != report.name:
+                    path.replace(out/path.name)
+            (stage/report.name).replace(report)
+        return result
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--year', type=int, choices=sorted(PAGES), required=True)
+    parser.add_argument('--out', required=True)
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument('--cache')
+    source.add_argument('--snapshots')
+    args = parser.parse_args(argv)
+    print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))

paper_atlas/collectors/public_openreview.py

@@ -27,11 +27,25 @@ RAW_ROOT = 'https://raw.githubusercontent.com/papercopilot/openreview/' + REVISI
 TREE_URL = 'https://api.github.com/repos/papercopilot/openreview/git/trees/' + REVISION + '?recursive=1'
 NEGATIVE = {'reject': 'rejected', 'rejected': 'rejected',
             'desk reject': 'desk_rejected', 'desk rejected': 'desk_rejected',
-            'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
+            'withdraw': 'withdrawn', 'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
 TRACKS = {'main': 'main', 'Position': 'position',
           'Datasets & Benchmarks': 'datasets_benchmarks'}
 WORKSHOP_REJECT_LABELS = {"aim-fm workshop @ neurips'24 reject",
                           'coginterp @ neurips 2025 reject'}
+# Exact curated labels, tied to the year and raw workshop track. The official
+# venue directory establishes workshop identity, not an original decision signature.
+ICLR_WORKSHOP_LABELS = {
+    'iclr 2024 workshop mefomo withdrawn submission':
+        (2024, 'workshop_me_fomo', 'withdrawn', 'ME-FoMo'),
+    'iclr 2024 workshop genai4dm withdrawn submission':
+        (2024, 'workshop_genai4dm', 'withdrawn', 'GenAI4DM'),
+    'iclr 2025 workshop agenticai reject':
+        (2025, 'workshop_agenticai', 'rejected', 'AgenticAI'),
+    'iclr 2025 workshop lmrl withdrawn submission':
+        (2025, 'workshop_lmrl', 'withdrawn', 'LMRL'),
+    'iclr 2026 workshop llm reasoning withdrawn submission':
+        (2026, 'workshop_llm_reasoning', 'withdrawn', 'LLM_Reasoning'),
+}
 
 
 def download(url, limit):
@@ -63,6 +77,20 @@ def normalize(row, family, year, provenance, include_workshops=False):
     """Return a normalized assertion or a reason requiring separate review."""
     raw_status = row.get('status', '')
     status = negative_status(raw_status)
+    workshop = ICLR_WORKSHOP_LABELS.get(' '.join(raw_status.casefold().split()))
+    workshop_metadata = {}
+    if workshop:
+        expected_year, expected_track, status, group_name = workshop
+        if family != 'iclr' or year != expected_year:
+            raise ValueError('Workshop status label conflicts with source conference year')
+        if row.get('track') != expected_track:
+            return None, 'conflicting_workshop_track'
+        if not include_workshops:
+            return None, 'separate_workshop'
+        workshop_metadata = {
+            'workshop_venueid': f'ICLR.cc/{year}/Workshop/{group_name}',
+            'workshop_identity_evidence_url': 'https://openreview.net/venue?id=ICLR.cc',
+        }
     if status is None:
         if re.search(r'\bretract\w*\b', raw_status, re.I):
             return None, 'retracted'
@@ -90,7 +118,9 @@ def normalize(row, family, year, provenance, include_workshops=False):
         return None, 'unmapped_venue'
     title = row.get('title')
     abstract = row.get('abstract')
-    if not isinstance(title, str) or not title.strip():
+    if title is None or (isinstance(title, str) and not title.strip()):
+        return None, 'missing_title'
+    if not isinstance(title, str):
         raise ValueError('Public submission lacks a title')
     if abstract is not None and not isinstance(abstract, str):
         raise ValueError('Abstract must be text or null')
@@ -116,7 +146,8 @@ def normalize(row, family, year, provenance, include_workshops=False):
             'venue': venue, 'year': year, 'track': normalized_track,
             'status': status, 'raw_decision': raw_status, 'evidence_url': evidence,
             'observed_at': AS_OF, 'tags': tags,
-            'metadata': {**provenance, 'raw_track': track, 'raw_status': raw_status,
+            'metadata': {**provenance, **workshop_metadata,
+                         'raw_track': track, 'raw_status': raw_status,
                          'evidence_kind': 'curated_public_status_assertion',
                          'original_decision_signature_verified': False,
                          'source_snapshot_upper_bound': AS_OF}}, None
@@ -215,7 +246,7 @@ def _collect_files(out, selected, include_workshops, fetcher):
             name = f'public-openreview-{family}-{year}-{track}.jsonl'
             path = stage / name
             path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
-            manifest = {'source': SOURCE, 'parser_version': 1, 'venue': records[0]['venue'],
+            manifest = {'source': SOURCE, 'parser_version': 3, 'venue': records[0]['venue'],
                         'year': year, 'track': track, 'count': len(records), 'status': 'partial',
                         'file': str(out / name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
                         'list_url': REPOSITORY + '/tree/' + REVISION, 'source_revision': REVISION,

paper_atlas/collectors/rss.py

@@ -0,0 +1,279 @@
+"""Reviewed RSS 2025 proceedings HTML; no PDF or private review access."""
+import argparse
+from collections import Counter
+from datetime import date
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import re
+import tempfile
+import unicodedata
+
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.io.html import Document, Node, clean
+from paper_atlas.io.http import fetch
+
+SOURCE = 'rss_proceedings'
+ROOT = 'https://www.roboticsproceedings.org/rss21/'
+HEADING = 'Robotics: Science and Systems XXI'
+PAGES = {'index': ROOT,
+         'accepted': 'https://roboticsconference.org/2025/program/papers/',
+         'demos': 'https://roboticsconference.org/2025/program/demos/',
+         'authorinfo': 'https://roboticsconference.org/2025/information/authorinfo/'}
+EXPECTED = 163
+DEMO_IDS = {2, 20, 21, 33, 58, 59, 62, 84, 86, 92, 93, 107, 129, 132, 141}
+
+
+def one(values, description):
+    if len(values) != 1:
+        raise ValueError('Expected one '+description)
+    return values[0]
+
+
+def comparable(text):
+    return unicodedata.normalize('NFC', clean(text))
+
+
+def parse_index(html):
+    document = Document(html)
+    nodes = list(document.root.walk())
+    if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
+        raise ValueError('Unexpected RSS proceedings edition')
+    result = {}
+    for row in (n for n in nodes if n.tag == 'tr'):
+        links = [n for n in row.walk() if n.tag == 'a'
+                 and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
+        if not links:
+            continue
+        link = one(links, 'proceedings entry link')
+        pid = int(link.attrs['href'][1:4])
+        title = clean(link.text())
+        authors = clean(one([n for n in row.walk() if n.tag == 'i'], 'index authors').text())
+        if pid in result or pid < 1 or not title or not authors:
+            raise ValueError('Empty or repeated RSS index entry')
+        result[pid] = {'title': title, 'authors_display': authors,
+                       'url': ROOT+link.attrs['href']}
+    occurrences = sum(n.tag == 'a' and bool(re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', '')))
+                      for n in nodes)
+    if not result or len(result) != occurrences:
+        raise ValueError('Unparsed or duplicate proceedings links')
+    return result
+
+
+def parse_program(html, heading, columns):
+    nodes = list(Document(html).root.walk())
+    if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [heading]:
+        raise ValueError('Unexpected RSS program page')
+    headers = [[clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag == 'th']
+               for row in nodes if row.tag == 'tr']
+    if list(columns) not in headers:
+        raise ValueError('Unexpected RSS program columns')
+    result = {}
+    for row in (n for n in nodes if n.tag == 'tr'):
+        cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
+        if not cells:
+            continue
+        if len(cells) != len(columns):
+            raise ValueError('Unexpected RSS program row width')
+        raw_id = clean(cells[0].text())
+        if not re.fullmatch('[1-9][0-9]*', raw_id):
+            raise ValueError('Invalid program ID')
+        pid = int(raw_id)
+        if pid in result:
+            raise ValueError('Duplicate program ID')
+        result[pid] = {key: clean(node.text()) for key, node in zip(columns, cells)}
+        if not result[pid]['Title']:
+            raise ValueError('Missing program title')
+    if not result:
+        raise ValueError('No RSS program rows')
+    return result
+
+
+def validate_program(index, accepted, demos, authorinfo):
+    if set(index) != set(accepted) or len(index) != EXPECTED or set(demos) != DEMO_IDS:
+        raise ValueError('RSS program and reviewed proceedings census differ')
+    text = clean(Document(authorinfo).root.text())
+    if ('There are two types of submissions to RSS' not in text
+            or 'Both types of papers appear in the RSS proceedings.' not in text):
+        raise ValueError('Submission category evidence changed')
+    for pid, item in index.items():
+        variants = [accepted[pid]['Title']] + ([demos[pid]['Title']] if pid in demos else [])
+        if any(comparable(title) != comparable(item['title']) for title in variants):
+            raise ValueError('Conflicting official program title for '+str(pid))
+
+
+def parse_paper(html, pid, entry, accepted, demo, evidence):
+    nodes = list(Document(html).root.walk())
+    meta = {}
+    for node in nodes:
+        if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
+            meta.setdefault(node.attrs['name'], []).append(node.attrs.get('content', ''))
+    title = one(meta.get('citation_title', []), 'citation title')
+    authors = meta.get('citation_author', [])
+    publication_date = one(meta.get('citation_publication_date', []), 'publication date')
+    if not re.fullmatch(r'2025/[0-9]{2}/[0-9]{2}', publication_date):
+        raise ValueError('Unexpected publication year/date')
+    date.fromisoformat(publication_date.replace('/', '-'))
+    if (one(meta.get('citation_conference_title', []), 'conference') != HEADING
+            or one(meta.get('citation_volume', []), 'volume') != '21'
+            or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
+            or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]
+            or comparable(title) != comparable(entry['title']) or not authors
+            or any(not clean(author) for author in authors)
+            or comparable(', '.join(authors)) != comparable(entry['authors_display'])):
+        raise ValueError('Proceedings/citation title, authors or edition disagree')
+    pdf = one(meta.get('citation_pdf_url', []), 'citation PDF URL')
+    if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
+        raise ValueError('Citation PDF ID differs; URL is metadata only')
+    bibtex = one([n.text() for n in nodes if n.tag == 'pre'], 'original BibTeX')
+    doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bibtex, re.I | re.M), 'BibTeX DOI')
+    year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bibtex, re.I | re.M), 'BibTeX year')
+    if doi != f'10.15607/RSS.2025.XXI.{pid:03d}' or year != '2025':
+        raise ValueError('BibTeX DOI/year differs from volume and paper ID')
+    abstract_parts = []
+    inside = False
+    markers = []
+    for node in (n for n in nodes if n.tag == 'p'):
+        labels = [clean(n.text()) for n in node.walk() if n.tag == 'b']
+        if 'Abstract:' in labels:
+            markers.append('start')
+            inside = True
+            value = clean(node.text()).removeprefix('Abstract:').strip()
+            if value:
+                abstract_parts.append(value)
+        elif 'Download:' in labels:
+            markers.append('end')
+            inside = False
+        elif inside:
+            abstract_parts.append(clean(node.text()))
+    if markers != ['start', 'end']:
+        raise ValueError('Abstract boundaries missing or repeated')
+    abstract = clean(' '.join(abstract_parts)) or None
+    track = 'demo' if demo else 'proceedings_unspecified'
+    tags = [{'namespace': SOURCE+':session', 'value': accepted['Session']}]
+    if demo:
+        tags.append({'namespace': SOURCE+':program_role', 'value': 'demonstration'})
+    return {'source': SOURCE, 'external_id': f'2025/p{pid:03d}', 'url': entry['url'],
+            'title': title, 'abstract': abstract, 'authors': authors,
+            'identifiers': {'doi': doi.lower()},
+            'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi.lower()}],
+            'venue': 'RSS', 'venue_aliases': [], 'year': 2025, 'track': track,
+            'status': 'accepted', 'observed_at': evidence['paper']['observed_at'],
+            'raw_decision': 'listed in official RSS 2025 proceedings and accepted program',
+            'tags': tags,
+            'metadata': {'publisher_paper_id': pid, 'citation_metadata': meta,
+                         'publication_date': publication_date, 'original_bibtex': bibtex,
+                         'index_entry': entry, 'accepted_program_entry': accepted,
+                         'demo_program_entry': demo, 'source_evidence': evidence,
+                         'missing_abstract_reason': None if abstract else 'empty_official_abstract_section',
+                         'track_scope_note': ('Explicit official demo program ID; not inferred from title.' if demo else
+                                              'Proceedings and accepted list do not individually identify submission type; session is not a track.'),
+                         'decision_date_precision': 'not_supplied',
+                         'publication_evidence': 'Published proceedings and accepted program; not an original decision note.'}}
+
+
+class Pages:
+    def __init__(self, *, cache=None, snapshots=None):
+        if bool(cache) == bool(snapshots):
+            raise ValueError('Choose exactly one cache or frozen snapshot directory')
+        self.cache = Path(cache) if cache else None
+        self.root = Path(snapshots).resolve() if snapshots else None
+        self.receipts = json.loads((self.root/'receipts.json').read_bytes())['receipts'] if self.root else []
+        self.evidence = {}
+        self.data = {}
+
+    def read(self, url):
+        if self.root:
+            receipt = one([r for r in self.receipts if r['url'] == url], 'exact frozen response receipt')
+            filename = Path(receipt['file'])
+            path = self.root/filename
+            if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(self.root)
+                    or receipt['http_status'] != 200 or receipt['content_type'] != 'text/html'
+                    or receipt.get('final_url', url) != url):
+                raise ValueError('Unsafe or unsuccessful frozen response')
+            data = path.read_bytes()
+            observed = receipt['at']
+        else:
+            data = fetch(url, self.cache, 2*1024**2, delay=1)
+            receipt = json.loads((self.cache/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_bytes())
+            observed = receipt['retrieved_at']
+        if (len(data) > 2*1024**2 or len(data) != receipt['bytes']
+                or hashlib.sha256(data).hexdigest() != receipt['sha256'] or receipt['url'] != url):
+            raise ValueError('Response checksum, URL or size differs')
+        evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
+                    'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
+                    'observed_at': timestamp(observed), 'http_status': 200, 'response_receipt': receipt}
+        self.evidence[url] = evidence
+        self.data[url] = data
+        return data.decode('utf-8'), evidence
+
+
+def collect(year, out, *, cache=None, snapshots=None):
+    if year != 2025:
+        raise ValueError('Only the independently reviewed RSS 2025 edition is supported')
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    with (out/'rss.collect.lock').open('a') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        report_path = out/'rss-2025.json'
+        if report_path.exists() or list(out.glob('rss-2025-*.jsonl')):
+            raise FileExistsError('Inspect existing output; never overwrite a completed or partial queue')
+        pages = Pages(cache=cache, snapshots=snapshots)
+        context = {key: pages.read(url) for key, url in PAGES.items()}
+        index = parse_index(context['index'][0])
+        accepted = parse_program(context['accepted'][0], 'Accepted Papers', ('ID', 'Session', 'Title', 'Authors'))
+        demos = parse_program(context['demos'][0], 'Demos', ('ID', 'Location', 'Title', 'Day', 'Time'))
+        validate_program(index, accepted, demos, context['authorinfo'][0])
+        records = []
+        for pid, entry in sorted(index.items()):
+            html, evidence = pages.read(entry['url'])
+            sources = {key: item[1] for key, item in context.items()}
+            sources['paper'] = evidence
+            records.append(parse_paper(html, pid, entry, accepted[pid], demos.get(pid), sources))
+        counts = Counter(r['track'] for r in records)
+        if counts != {'demo': len(DEMO_IDS), 'proceedings_unspecified': EXPECTED-len(DEMO_IDS)}:
+            raise ValueError('Track census differs; review official source changes')
+        files = []
+        with tempfile.TemporaryDirectory(prefix='.rss-', dir=out) as temporary:
+            stage = Path(temporary)
+            (stage/'evidence').mkdir()
+            for url, data in pages.data.items():
+                (stage/pages.evidence[url]['file']).write_bytes(data)
+            for track in sorted(counts):
+                group = [r for r in records if r['track'] == track]
+                path = stage/f'rss-2025-{track}.jsonl'
+                path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
+                manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2025,
+                            'track': track, 'count': len(group), 'file': str(out/path.name),
+                            'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
+                            'list_url': ROOT, 'status': 'partial',
+                            'missing_abstract_count': sum(r['abstract'] is None for r in group)}
+                path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
+                files.append(manifest)
+            result = {'source': SOURCE, 'year': 2025, 'records': len(records), 'files': files,
+                      'source_evidence': list(pages.evidence.values()), 'database_written': False,
+                      'pdfs_downloaded': False, 'status': 'collected_pending_identity_review',
+                      'note': 'Only the published proceedings. Demo IDs have explicit program evidence; '
+                              'other individual submission types remain unknown. No absent-based negative decisions. '
+                              'Separate source identity review and independent production acceptance are required.'}
+            (stage/report_path.name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
+            (out/'evidence').mkdir(exist_ok=True)
+            for path in (stage/'evidence').iterdir():
+                path.replace(out/'evidence'/path.name)
+            for path in stage.iterdir():
+                if path.is_file() and path.name != report_path.name:
+                    path.replace(out/path.name)
+            (stage/report_path.name).replace(report_path)
+        return result
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--year', type=int, choices=[2025], required=True)
+    parser.add_argument('--out', required=True)
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument('--cache')
+    group.add_argument('--snapshots')
+    args = parser.parse_args(argv)
+    print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))

paper_atlas/collectors/vision.py

@@ -7,7 +7,7 @@ import re
 import time
 from datetime import date
 from urllib.error import HTTPError
-from urllib.parse import parse_qs, unquote, urljoin, urlsplit
+from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit
 
 from paper_atlas.io.http import fetch
 from paper_atlas.io.html import Document, Node, clean
@@ -147,7 +147,8 @@ def read_index(index,cache):
     links=set()
     for node in Document(html).root.walk():
         if node.tag!='a':continue
-        link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[])
+        # Fragments select positions within a page, not separate HTTP resources.
+        link=urldefrag(urljoin(url,node.attrs.get('href','')))[0];day=parse_qs(urlsplit(link).query).get('day',[])
         if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue
         if day[0]!='all':
             try:valid=date.fromisoformat(day[0]).year==index['year']

paper_atlas/collectors/www_archive.py

@@ -0,0 +1,340 @@
+"""Reproducible early WWW metadata; unresolved program entries stay outside imports."""
+import argparse
+from collections import defaultdict
+import hashlib
+from html.parser import HTMLParser
+import json
+from pathlib import Path
+import re
+import tempfile
+from urllib.parse import urljoin, urlsplit
+
+from paper_atlas.io.html import Document, clean
+from paper_atlas.io.http import fetch
+
+ROOT = 'https://archives.iw3c2.org/'
+SPRING = ROOT+'www3/proceedings/'
+FALL = ROOT+'www2/Proceedings/WWW2_Proceedings.html'
+SEEDS = [ROOT+'www1/PrelimProcs.html', ROOT+'www1/Proceedings.html', FALL,
+         SPRING+'proceedings.html', *[SPRING+'papers/'+x+'.html' for x in ('author','session','abstract')],
+         *[SPRING+'posters/'+x+'.html' for x in ('author','abstract')]]
+
+
+def decode(raw):
+    try:
+        return raw.decode('utf-8-sig'), 'utf-8'
+    except UnicodeDecodeError:
+        return raw.decode('iso-8859-1'), 'iso-8859-1 fallback; original bytes retained'
+
+
+def text(html):
+    return re.sub(r'[ \t\r\n\f\v]+',' ',Document(html).root.text()).strip(' \t\r\n\f\v')
+
+
+def anchors(html):
+    return [(n.attrs.get('href',''),clean(n.text())) for n in Document(html).root.walk() if n.tag=='a']
+
+
+def source_url(url):
+    p=urlsplit(url)
+    if p.scheme!='https' or p.netloc!='archives.iw3c2.org' or p.query or p.fragment or not p.path.endswith('.html'):
+        raise ValueError('Only exact official archive HTML source URLs are allowed')
+    return url
+
+
+class Sources:
+    def __init__(self, directory, expected_sha256):
+        self.root=Path(directory).resolve();manifest=self.root/'manifest.json';raw=manifest.read_bytes()
+        if hashlib.sha256(raw).hexdigest()!=expected_sha256:
+            raise ValueError('Source cache manifest checksum mismatch')
+        self.rows={};self.used=set()
+        for row in json.loads(raw)['sources']:
+            url=source_url(row['url'])
+            if url in self.rows or row.get('status')!=200:
+                raise ValueError('Duplicate or incomplete source response')
+            p=self.root/row['file']
+            if p.is_symlink() or p.parent!=self.root or not p.is_file():
+                raise ValueError('Source cache paths must be plain files directly within the cache')
+            data=p.read_bytes()
+            if len(data)>2*1024**2 or len(data)!=row['bytes'] or hashlib.sha256(data).hexdigest()!=row['sha256']:
+                raise ValueError('Source response checksum or size mismatch')
+            self.rows[url]={'raw':data,'receipt':row}
+
+    def read(self,url):
+        source_url(url)
+        if url not in self.rows:
+            raise ValueError('Required source missing from cache: '+url)
+        self.used.add(url)
+        return decode(self.rows[url]['raw'])[0]
+
+    def proof(self,urls):
+        return [{'url':url,'sha256':self.rows[url]['receipt']['sha256'],
+                 'encoding':decode(self.rows[url]['raw'])[1]} for url in sorted(set(urls))]
+
+
+def download_sources(http_cache, directory):
+    directory=Path(directory)
+    if directory.exists():
+        raise ValueError('Refusing to overwrite a source snapshot')
+    directory.mkdir(parents=True)
+    responses={url:fetch(url,http_cache,2*1024**2) for url in SEEDS}
+    html=decode(responses[FALL])[0]
+    for href,label in anchors(html):
+        if href!='Author_Inst.html' and href.endswith('.html'):
+            url=source_url(urljoin(FALL,href))
+            if not url.startswith(ROOT+'www2/Proceedings/'):
+                raise ValueError('Unexpected fall category source')
+            responses[url]=fetch(url,http_cache,2*1024**2)
+    receipts=[]
+    for i,(url,raw) in enumerate(sorted(responses.items())):
+        name='source-'+str(i).zfill(2)+'.html';(directory/name).write_bytes(raw)
+        receipts.append({'url':url,'status':200,'file':name,'bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()})
+    manifest=directory/'manifest.json';manifest.write_text(json.dumps({'sources':receipts},indent=2)+'\n')
+    return hashlib.sha256(manifest.read_bytes()).hexdigest()
+
+
+def identity(href,track):
+    url=urljoin(SPRING+track+'/',href);p=urlsplit(url)
+    match=re.fullmatch(r'/www3/proceedings/'+track+r'/(\d+)/[^?#]+',p.path)
+    if p.scheme=='https' and p.netloc=='archives.iw3c2.org' and not p.query and not p.fragment and match:
+        return match[1],url
+    return None
+
+
+class AuthorIndex(HTMLParser):
+    def __init__(self,html,track):
+        super().__init__(convert_charrefs=True);self.track=track;self.name='';self.bold=None;self.link=None;self.link_text=[]
+        self.entries=defaultdict(lambda:{'authors':[],'urls':set(),'titles':set()});self.feed(html)
+
+    def handle_starttag(self,tag,attrs):
+        if tag=='b':self.bold=[]
+        if tag=='a':self.link=dict(attrs).get('href');self.link_text=[]
+
+    def handle_data(self,data):
+        if self.bold is not None:self.bold.append(data)
+        if self.link is not None:self.link_text.append(data)
+
+    def handle_endtag(self,tag):
+        if tag=='b' and self.bold is not None:
+            self.name=clean(''.join(self.bold));self.bold=None
+        if tag=='a' and self.link is not None:
+            own=identity(self.link,self.track)
+            if own:
+                key,url=own;entry=self.entries[key];entry['urls'].add(url);entry['titles'].add(clean(''.join(self.link_text)))
+                if not self.name:raise ValueError('Paper link has no archive author context')
+                if self.name not in entry['authors']:entry['authors'].append(self.name)
+            self.link=None
+
+
+def sections(html):
+    """Use explicit h3 boundaries; old HTML omits many p/li closing tags."""
+    result=[]
+    for chunk in re.split(r'<h3\b[^>]*>',html,flags=re.I)[1:]:
+        parts=re.split(r'</h3\s*>',chunk,maxsplit=1,flags=re.I)
+        if len(parts)!=2:raise ValueError('Unclosed archive entry heading')
+        heading,body=parts
+        body=re.split(r'<(?:hr|h2|address)\b',body,maxsplit=1,flags=re.I)[0]
+        author=re.search(r'<i\b[^>]*>(.*?)</i\s*>',body,re.I|re.S)
+        paragraphs=[]
+        for paragraph in re.findall(r'<p\b[^>]*>(.*?)(?=</p\s*>|<p\b|$)',body,re.I|re.S):
+            value=text(paragraph)
+            if value and not value.casefold().startswith('keywords:'):paragraphs.append(value)
+        result.append({'heading':heading,'title':text(heading),'abstract':'\n\n'.join(paragraphs)or None,
+                       'author_byline_raw':text(author[1])if author else None})
+    if not result:raise ValueError('No explicit archive entry headings')
+    return result
+
+
+def reviewed_posters(sources, path, expected_sha256, cache_sha256):
+    if path is None:
+        if expected_sha256:raise ValueError('Reviewed poster path required')
+        return {},None
+    raw=Path(path).read_bytes()
+    if hashlib.sha256(raw).hexdigest()!=expected_sha256:raise ValueError('Reviewed poster checksum mismatch')
+    review=json.loads(raw)
+    if set(review)!={'format','source_cache_sha256','records'} or review['format']!=1 or review['source_cache_sha256']!=cache_sha256:
+        raise ValueError('Reviewed poster snapshot binding differs')
+    if not isinstance(review['records'],list) or not 0<len(review['records'])<=100:
+        raise ValueError('Bounded nonempty poster review required')
+    result={}
+    for row in review['records']:
+        if set(row)!={'entry_id','individual_url','individual_sha256','title','authors','author_source','reason','missing_abstract_reason'}:
+            raise ValueError('Unexpected reviewed poster fields')
+        key=row['entry_id'];own=identity(row['individual_url'],'posters')
+        if not isinstance(key,str) or not own or own[0]!=key or key in result:
+            raise ValueError('Reviewed poster identity duplicated or outside canonical entry')
+        if row['author_source'] not in ('canonical_byline','author_index') or not isinstance(row['authors'],list) or not row['authors'] or any(not isinstance(a,str) or not a.strip()for a in row['authors']):
+            raise ValueError('Explicit author evidence required')
+        if len(set(row['authors']))!=len(row['authors']) or not isinstance(row['reason'],str) or not row['reason'].strip():
+            raise ValueError('Reviewed author order/reason invalid')
+        if sources.rows.get(own[1],{}).get('receipt',{}).get('sha256')!=row['individual_sha256']:
+            raise ValueError('Individual poster page checksum differs')
+        page=sources.read(own[1]);headings=re.findall(r'<h1\b[^>]*>(.*?)</h1\s*>',page,re.I|re.S)
+        if len(headings)!=1 or text(headings[0])!=row['title']:
+            raise ValueError('Reviewed title differs from exact canonical H1')
+        header=re.split(r'</h1\s*>',page,maxsplit=1,flags=re.I)[1]
+        header=re.split(r'<h[23]\b',header,maxsplit=1,flags=re.I)[0]
+        header=re.split(r'\b(?:keywords|abstract)\s*:',text(header),maxsplit=1,flags=re.I)[0]
+        if row['author_source']=='canonical_byline' and any(a.casefold() not in header.casefold()for a in row['authors']):
+            raise ValueError('Reviewed author absent from canonical byline region')
+        result[key]={**row,'canonical_byline_region_raw':header}
+    return result,hashlib.sha256(raw).hexdigest()
+
+
+def reviewed_row(sources,key,url,item,authors,review):
+    if url!=review['individual_url']:
+        raise ValueError('Reviewed canonical page differs from abstract-entry URL')
+    if authors and authors['urls']!={url}:
+        raise ValueError('Reviewed poster author-index URL disagrees')
+    if review['author_source']=='author_index':
+        names=[]
+        for name in authors['authors']if authors else []:
+            parts=name.split(',');names.append(clean(parts[1]+' '+parts[0])if len(parts)==2 else name)
+        if names!=review['authors']:raise ValueError('Reviewed author-index evidence differs')
+    missing=review['missing_abstract_reason']
+    if not isinstance(missing,str) or bool(missing.strip())==bool(item['abstract']):
+        raise ValueError('Missing abstract requires explicit reason; never invent a summary')
+    urls=[SPRING+'posters/author.html',SPRING+'posters/abstract.html',url]
+    return {'source':'www_archive','external_id':'www3:posters:'+key,'url':url,'title':review['title'],
+            'abstract':item['abstract'],'authors':review['authors'],'identifiers':{},'venue':'WWW','year':1995,
+            'track':'poster','event_key':'www-1995-spring','status':'accepted','raw_decision':'listed in official posters proceedings',
+            'metadata':{'entry_id':key,'source_evidence':sources.proof(urls),'abstract_heading_title_raw':item['title'],
+                        'author_byline_raw':item['author_byline_raw'],'author_index_raw':authors['authors']if authors else [],
+                        'author_index_titles_raw':sorted(authors['titles'])if authors else [],'abstract_href_raw':url,
+                        'author_order':review['author_source'],'reviewed_poster_identity':review,
+                        'missing_abstract_reason':missing or None},
+            'tags':[{'namespace':'www_archive:proceedings_section','value':'posters'}]}
+
+
+def spring_rows(sources,track,reviews=None):
+    base=SPRING+track+'/';author_url=base+'author.html';abstract_url=base+'abstract.html'
+    authors=AuthorIndex(sources.read(author_url),track).entries
+    listing=sections(sources.read(abstract_url));by_id={};held=[];session={}
+    if track=='papers':
+        session_url=base+'session.html'
+        for href,title in anchors(sources.read(session_url)):
+            own=identity(href,track)
+            if own:
+                key,url=own
+                if key in session:raise ValueError('Duplicate main session entry ID')
+                session[key]=url
+    for item in listing:
+        links=[identity(h,track)for h,t in anchors(item['heading'])if identity(h,track)]
+        if len(links)!=1:
+            held.append({'reason':'unverified_heading_identity','evidence':abstract_url,'candidate':item});continue
+        key,url=links[0]
+        if key in by_id:raise ValueError('Duplicate abstract entry ID')
+        by_id[key]=(url,item)
+    if not by_id:raise ValueError('No stable abstract entry IDs')
+    if track=='papers' and (set(authors)!=set(by_id) or set(session)!=set(by_id)):
+        raise ValueError('Main author/session/abstract entry ID sets disagree')
+    if set(authors)-set(by_id):raise ValueError('Author index has entries outside the abstract listing')
+    reviews=reviews or {}
+    if set(reviews)-set(by_id):raise ValueError('Reviewed poster outside abstract listing')
+    rows=[]
+    for key,(url,item) in by_id.items():
+        if key in reviews:
+            if track!='posters':raise ValueError('Reviews only apply to poster entries')
+            rows.append(reviewed_row(sources,key,url,item,authors.get(key),reviews[key]));continue
+        if key not in authors:
+            held.append({'reason':'abstract_entry_missing_linked_author_index','entry_id':key,'evidence':abstract_url,'candidate':item});continue
+        alternatives=authors[key]['urls']
+        if len(alternatives)!=1:raise ValueError('Ambiguous author-index paper URL')
+        canonical=next(iter(alternatives))
+        if track=='papers' and session[key]!=canonical:raise ValueError('Main author/session paper URLs disagree')
+        if url!=canonical and (track!='papers' or not re.search(r'\s',url)):
+            raise ValueError('Uncorroborated archive URL discrepancy')
+        if not item['abstract']:
+            held.append({'reason':'missing_abstract','entry_id':key,'evidence':abstract_url,'candidate':item});continue
+        if item['title'].casefold() not in {t.casefold() for t in authors[key]['titles']}:
+            held.append({'reason':'abstract_and_author_index_titles_disagree','entry_id':key,'evidence':abstract_url,'candidate':item,'author_index_titles':sorted(authors[key]['titles'])});continue
+        names=[]
+        for name in authors[key]['authors']:
+            parts=name.split(',')
+            names.append(clean(parts[1]+' '+parts[0])if len(parts)==2 else name)
+        proof_urls=[author_url,abstract_url]+([session_url]if track=='papers'else[])
+        rows.append({'source':'www_archive','external_id':'www3:'+track+':'+key,'url':canonical,
+                     'title':item['title'],'abstract':item['abstract'],'authors':names,'identifiers':{},
+                     'venue':'WWW','year':1995,'track':'main'if track=='papers'else'poster','event_key':'www-1995-spring',
+                     'status':'accepted','raw_decision':'listed in official '+track+' proceedings',
+                     'metadata':{'entry_id':key,'source_evidence':sources.proof(proof_urls),
+                                 'author_index_raw':authors[key]['authors'],'author_index_titles_raw':sorted(authors[key]['titles']),'author_order':'archive author-index order, not original byline rank',
+                                 'author_byline_raw':item['author_byline_raw'],'abstract_href_raw':url,
+                                 'url_corroborated_by_author_and_session':url!=canonical},
+                     'tags':[{'namespace':'www_archive:proceedings_section','value':track}]})
+    return rows,held,{'abstract_entries':len(by_id),'linked_author_entries':len(authors),'ready':len(rows),'held':len(held)}
+
+
+def early_review(sources):
+    held=[];url=ROOT+'www1/PrelimProcs.html';html=sources.read(url)
+    for ordinal,chunk in enumerate(re.split(r'<li\b[^>]*>',html,flags=re.I)[1:],1):
+        title=re.search(r'<em>Title:</em>\s*<b>(.*?)</b>',chunk,re.I|re.S)
+        if not title:raise ValueError('Unclassified first-conference entry')
+        names=re.findall(r'<em>Author:</em>(.*?)(?=<br\b)',chunk,re.I|re.S)
+        held.append({'reason':'formal_presentation_scope_and_count_need_review','event_key':'www-1994-spring',
+                     'ordinal':ordinal,'evidence':url,'candidate':{'title':text(title[1]),
+                     'authors_raw':[re.sub(r'\s*<[^>]+>','',text(n)).strip()for n in names],
+                     'keywords_raw':text(re.search(r'<em>Keywords:</em>(.*?)(?=<br\b)',chunk,re.I|re.S)[1]) if re.search(r'<em>Keywords:</em>(.*?)(?=<br\b)',chunk,re.I|re.S)else None},
+                     'source_evidence':sources.proof([url])})
+    if not held:raise ValueError('First-conference presentation listing empty')
+    category_urls={}
+    for href,label in anchors(sources.read(FALL)):
+        if href!='Author_Inst.html' and href.endswith('.html'):
+            category_url=source_url(urljoin(FALL,href))
+            if not category_url.startswith(ROOT+'www2/Proceedings/'):raise ValueError('Unexpected fall category')
+            category_urls[category_url]=label
+    if not category_urls:raise ValueError('Fall archive category listing empty')
+    for url,label in sorted(category_urls.items()):
+        html=sources.read(url)
+        for ordinal,chunk in enumerate(re.split(r'<dt\b[^>]*>',html,flags=re.I)[1:],1):
+            fields=re.split(r'<dd\b[^>]*>',chunk,maxsplit=1,flags=re.I)
+            title=text(fields[0]);raw_byline=text(re.split(r'</dl\b',fields[1],maxsplit=1,flags=re.I)[0]) if len(fields)==2 else None
+            if title:
+                held.append({'reason':'mixed_program_category_not_verified_paper_acceptance','event_key':'www-1994-fall',
+                             'category':label,'ordinal':ordinal,'evidence':url,'candidate':{'title':title,'authors_raw':raw_byline},
+                             'source_evidence':sources.proof([FALL,url])})
+    return held,{'first_conference_fields':sum(r['event_key']=='www-1994-spring'for r in held),'fall_categories':len(category_urls),
+                 'fall_program_entries':sum(r['event_key']=='www-1994-fall'for r in held)}
+
+
+def collect(source_cache,cache_sha256,out,*,reviewed=None,reviewed_sha256=None):
+    out=Path(out)
+    if out.exists():raise ValueError('Refusing to overwrite an existing normalized batch')
+    sources=Sources(source_cache,cache_sha256);groups={};held=[];counts={}
+    reviews,review_sha=reviewed_posters(sources,reviewed,reviewed_sha256,cache_sha256)
+    for track in ('papers','posters'):
+        rows,review,counts[track]=spring_rows(sources,track,reviews if track=='posters'else None);groups[track]=rows;held.extend(review)
+    review,counts['earlier']=early_review(sources);held.extend(review)
+    for item in held:
+        if 'source_evidence' not in item:item['source_evidence']=sources.proof([item['evidence']])
+    out.parent.mkdir(parents=True,exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix='.www-archive-',dir=out.parent)as tmp:
+        stage=Path(tmp);queue=[]
+        for track,rows in groups.items():
+            if not rows:continue
+            filename='www-1995-spring-'+('main'if track=='papers'else'poster')+'.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in rows).encode();(stage/filename).write_bytes(raw)
+            manifest={'source':'www_archive','parser_version':2 if review_sha else 1,'venue':'WWW','year':1995,'track':rows[0]['track'],'event_key':'www-1995-spring',
+                      'list_url':SPRING+track+'/abstract.html','count':len(rows),'status':'partial','file_sha256':hashlib.sha256(raw).hexdigest(),
+                      'missing_abstract_count':sum(not bool((r['abstract']or '').strip())for r in rows),'source_listing_count':counts[track]['abstract_entries'],'held_entry_count':counts[track]['held'],'source_cache_sha256':cache_sha256,'scope':'Verified linked archive entries; held entries and other event tracks excluded, not complete event coverage.'}
+            (stage/filename).with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');queue.append(str(out/filename))
+        (stage/'review-candidates.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in held))
+        report={'status':'normalized_not_imported','source_cache_sha256':cache_sha256,'counts':counts,'ready':sum(len(r)for r in groups.values()),'held':len(held),
+                'source_evidence':sources.proof(sources.used),'files':queue,'scope':'Early WWW archive only; no PDF fetching, no rejected/withdrawn inference, no database access. Event-aware importer required.'}
+        if review_sha:report['reviewed_posters_sha256']=review_sha
+        (stage/'report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
+        (stage/'import-queue.json').write_text(json.dumps({'files':queue},indent=2)+'\n')
+        stage.rename(out)
+    return report
+
+
+def main(argv=None,*,prog=None):
+    p=argparse.ArgumentParser(prog=prog);p.add_argument('--source-cache',required=True);p.add_argument('--source-cache-sha256');p.add_argument('--http-cache');p.add_argument('--out',required=True)
+    p.add_argument('--reviewed-posters');p.add_argument('--reviewed-posters-sha256')
+    args=p.parse_args(argv)
+    if bool(args.http_cache)==bool(args.source_cache_sha256):p.error('Choose a fresh --http-cache download or an existing --source-cache-sha256 snapshot')
+    sha=download_sources(args.http_cache,args.source_cache)if args.http_cache else args.source_cache_sha256
+    if bool(args.reviewed_posters)!=bool(args.reviewed_posters_sha256):p.error('Reviewed poster file and SHA256 must be supplied together')
+    print(json.dumps(collect(args.source_cache,sha,args.out,reviewed=args.reviewed_posters,reviewed_sha256=args.reviewed_posters_sha256),ensure_ascii=False),flush=True)
+
+
+if __name__=='__main__':main()

paper_atlas/curation/__init__.py

@@ -0,0 +1 @@
+"""Evidence-bound identity preparation and independent type/tag annotations."""

paper_atlas/curation/annotations.py

@@ -0,0 +1,218 @@
+"""Append-only tag/type assertions and explicit retractions; no inferred labels."""
+import argparse
+from contextlib import closing
+from importlib.resources import files
+import json
+import re
+from urllib.parse import urlparse
+
+from paper_atlas.db.backend import database_arguments, inserted_id, open_read_connection, open_store
+from paper_atlas.db.submissions import timestamp
+
+TABLE = 'annotation_events'
+MIGRATION = 'annotation_events_v1'
+COLUMNS = ('id', 'request_id', 'paper_id', 'original_paper_id', 'kind', 'namespace',
+           'value', 'source_record_id', 'evidence_key', 'evidence_url', 'reason',
+           'actor', 'observed_at', 'retracts_id')
+ACTIVE = '''a.retracts_id IS NULL AND NOT EXISTS
+    (SELECT 1 FROM annotation_events removed WHERE removed.retracts_id=a.id)'''
+
+
+def available(conn):
+    if getattr(conn, 'postgres', False):
+        if hasattr(conn, 'has_table'):
+            return conn.has_table(TABLE)
+        return bool(conn.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
+    return bool(conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
+
+
+def _begin_write(conn):
+    if not getattr(conn, 'postgres', False) and not conn.in_transaction:
+        conn.execute('BEGIN IMMEDIATE')
+
+
+def initialize(conn):
+    """Explicit transactional schema extension; does not rescan or rewrite papers."""
+    _begin_write(conn)
+    exists = available(conn)
+    marker = conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MIGRATION,)).fetchone()
+    if exists != bool(marker):
+        raise RuntimeError('Annotation table/upgrade marker mismatch; inspect before changing schema')
+    if exists:
+        result = conn.execute('SELECT * FROM annotation_events LIMIT 0')
+        description = result.cursor.description if getattr(conn, 'postgres', False) else result.description
+        if tuple(d[0] for d in description) != COLUMNS:
+            raise RuntimeError('Unexpected annotation schema')
+        return False
+    ddl = files('paper_atlas.db').joinpath('annotations.sql').read_text()
+    if getattr(conn, 'postgres', False):
+        ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
+        ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
+        ddl = ddl.replace('id BIGINT PRIMARY KEY', 'id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY', 1)
+    for statement in ddl.split(';'):
+        if statement.strip():
+            conn.execute(statement)
+    conn.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MIGRATION,))
+    return True
+
+
+def _text(value, name, limit=512):
+    if not isinstance(value, str) or not value.strip() or len(value) > limit:
+        raise ValueError(name + ' must be nonempty text of at most ' + str(limit) + ' characters')
+    return value
+
+
+def _id(value, name):
+    if type(value) is not int or value <= 0:
+        raise ValueError(name + ' must be a positive integer')
+    return value
+
+
+def _url(value):
+    if value is not None:
+        _text(value, 'evidence_url', 4096)
+        parsed = urlparse(value)
+        if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password:
+            raise ValueError('evidence_url must be an HTTP(S) URL without credentials')
+    return value
+
+
+def _require(conn):
+    if not available(conn) or not conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MIGRATION,)).fetchone():
+        raise RuntimeError('Run paper-atlas annotate init explicitly before adding annotations')
+
+
+def _row(row):
+    return dict(zip(COLUMNS, (row[i] for i in range(len(COLUMNS)))))
+
+
+def _replay(conn, request_id, payload):
+    old = conn.execute('SELECT * FROM annotation_events WHERE request_id=?', (request_id,)).fetchone()
+    if old is None:
+        return None
+    old = _row(old)
+    if any(old[k] != value for k, value in payload.items()):
+        raise ValueError('request_id already identifies different annotation content')
+    return old['id']
+
+
+def assert_annotation(conn, paper, kind, namespace, value, *, request_id, evidence_key,
+                      reason, actor, evidence_url=None, source_record_id=None):
+    _begin_write(conn)
+    _require(conn)
+    paper = _id(paper, 'paper')
+    if kind not in ('tag', 'research_type'):
+        raise ValueError('kind must be tag or research_type')
+    _text(namespace, 'namespace', 128)
+    if not re.fullmatch(r'[a-z][a-z0-9_]*:[a-z][a-z0-9_.-]*', namespace):
+        raise ValueError('namespace must have an explicit source:scope')
+    payload = dict(original_paper_id=paper, kind=kind, namespace=namespace,
+                   value=_text(value, 'value'), request_id=_text(request_id, 'request_id', 128),
+                   source_record_id=source_record_id, evidence_key=_text(evidence_key, 'evidence_key', 256),
+                   reason=_text(reason, 'reason', 8192), actor=_text(actor, 'actor', 256),
+                   evidence_url=_url(evidence_url), retracts_id=None)
+    if source_record_id is not None:
+        _id(source_record_id, 'source_record_id')
+    replay = _replay(conn, request_id, payload)
+    if replay is not None:
+        return replay
+    if not conn.execute('SELECT 1 FROM papers WHERE id=?', (paper,)).fetchone():
+        raise ValueError('Existing canonical paper required; no automatic identity merge')
+    prefix = namespace.split(':', 1)[0]
+    if prefix != 'custom' and source_record_id is None:
+        raise ValueError('Platform annotations require their own source record')
+    if source_record_id is not None:
+        source = conn.execute('''SELECT r.paper_id,s.name FROM source_records r
+            JOIN sources s ON s.id=r.source_id WHERE r.id=?''', (source_record_id,)).fetchone()
+        if not source or source[0] != paper or (prefix != 'custom' and prefix != source[1]):
+            raise ValueError('Annotation source record/namespace must belong to this paper and platform')
+    payload.update(paper_id=paper, observed_at=timestamp())
+    fields = tuple(payload)
+    return inserted_id(conn, 'INSERT INTO annotation_events(' + ','.join(fields) + ') VALUES(' +
+                       ','.join('?' for _ in fields) + ')', tuple(payload[k] for k in fields))
+
+
+def retract_annotation(conn, event, *, request_id, reason, actor, evidence_url=None, custom_only=False):
+    _begin_write(conn)
+    _require(conn)
+    event = _id(event, 'event')
+    payload = dict(request_id=_text(request_id, 'request_id', 128), retracts_id=event,
+                   reason=_text(reason, 'reason', 8192), actor=_text(actor, 'actor', 256), evidence_url=_url(evidence_url))
+    target = conn.execute('SELECT * FROM annotation_events WHERE id=?', (event,)).fetchone()
+    if target is None or target[-1] is not None:
+        raise ValueError('Retraction must reference an existing assertion')
+    target = _row(target)
+    if custom_only and not target['namespace'].startswith('custom:'):
+        raise ValueError('Manual CLI cannot retract platform evidence')
+    replay = _replay(conn, request_id, payload)
+    if replay is not None:
+        return replay
+    if conn.execute('SELECT 1 FROM annotation_events WHERE retracts_id=?', (event,)).fetchone():
+        raise ValueError('Assertion already retracted; its history is retained')
+    payload.update({k: target[k] for k in ('paper_id', 'original_paper_id', 'kind', 'namespace', 'value', 'source_record_id', 'evidence_key')})
+    payload['observed_at'] = timestamp()
+    fields = tuple(payload)
+    return inserted_id(conn, 'INSERT INTO annotation_events(' + ','.join(fields) + ') VALUES(' +
+                       ','.join('?' for _ in fields) + ')', tuple(payload[k] for k in fields))
+
+
+def observations(conn, paper, *, kind=None, history=False):
+    if not available(conn):
+        return []
+    where = ['a.paper_id=?']; params = [_id(paper, 'paper')]
+    if not history:
+        where.append(ACTIVE)
+    if kind is not None:
+        if kind not in ('tag', 'research_type'):
+            raise ValueError('Invalid annotation kind')
+        where.append('a.kind=?'); params.append(kind)
+    return [_row(r) for r in conn.execute('SELECT a.* FROM annotation_events a WHERE ' +
+                                         ' AND '.join(where) + ' ORDER BY a.id', params)]
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog, description='显式初始化、追加或撤回带证据的自定义标签和研究类型')
+    actions = parser.add_subparsers(dest='action', required=True)
+    for action in ('init', 'assert', 'retract', 'list'):
+        p = actions.add_parser(action)
+        database_arguments(p)
+        if action in ('assert', 'retract'):
+            for field in ('request-id', 'reason', 'actor'):
+                p.add_argument('--' + field, required=True)
+            p.add_argument('--evidence-url')
+        if action in ('assert', 'list'):
+            p.add_argument('--paper', type=int, required=True)
+            p.add_argument('--kind', choices=('tag', 'research_type'), required=action == 'assert')
+        if action == 'assert':
+            for field in ('namespace', 'value', 'evidence-key'):
+                p.add_argument('--' + field, required=True)
+            p.add_argument('--source-record', type=int)
+        if action == 'retract':
+            p.add_argument('--event', type=int, required=True)
+        if action == 'list':
+            p.add_argument('--history', action='store_true')
+    args = parser.parse_args(argv)
+    if args.action == 'list':
+        with closing(open_read_connection(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema)) as conn:
+            result = observations(conn, args.paper, kind=args.kind, history=args.history)
+    else:
+        if args.action == 'assert' and not args.namespace.startswith('custom:'):
+            parser.error('Manual CLI requires custom:* namespace; platform assertions use validated source APIs')
+        with closing(open_store(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema).conn) as conn:
+            try:
+                if not getattr(conn, 'postgres', False):
+                    conn.commit()
+                    conn.execute('BEGIN IMMEDIATE')
+                if args.action == 'init':
+                    result = {'initialized': initialize(conn)}
+                else:
+                    kwargs = dict(request_id=args.request_id, reason=args.reason, actor=args.actor, evidence_url=args.evidence_url)
+                    event = (assert_annotation(conn, args.paper, args.kind, args.namespace, args.value,
+                              evidence_key=args.evidence_key, source_record_id=args.source_record, **kwargs)
+                             if args.action == 'assert' else retract_annotation(conn, args.event, custom_only=True, **kwargs))
+                    result = {'event_id': event}
+                conn.commit()
+            except BaseException:
+                conn.rollback()
+                raise
+    print(json.dumps(result, ensure_ascii=False, indent=2))

paper_atlas/curation/identifiers.py

@@ -0,0 +1,212 @@
+"""Add reviewed stable identifiers without rewriting bibliographic source fields.
+
+Hashes establish file integrity, not the truth of an identity assertion. Identity
+review and a fresh database preflight remain separate requirements.
+"""
+import argparse
+from datetime import datetime
+import hashlib
+import json
+from pathlib import Path
+import re
+import tempfile
+from urllib.parse import urlsplit
+
+
+SCOPE = 'input_and_evidence_integrity_only'
+MAX_METADATA_BYTES = 16 * 1024 * 1024
+
+
+def digest(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+def object_keys(value, required, optional=()):
+    if not isinstance(value, dict) or not set(required) <= value.keys() or value.keys() - set(required) - set(optional):
+        raise ValueError('Missing or unsupported plan fields')
+
+
+def json_bytes(value):
+    return (json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n').encode('utf-8')
+
+
+def decode(raw):
+    def unique(pairs):
+        result = {}
+        for key, value in pairs:
+            if key in result:
+                raise ValueError('Duplicate JSON key: ' + key)
+            result[key] = value
+        return result
+    return json.loads(raw, object_pairs_hook=unique,
+                      parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Non-finite JSON value')))
+
+
+def metadata(path, expected=None):
+    with Path(path).open('rb') as stream:
+        raw = stream.read(MAX_METADATA_BYTES + 1)
+    if len(raw) > MAX_METADATA_BYTES:
+        raise ValueError('Metadata exceeds size limit')
+    if expected is not None:
+        check_hash(raw, expected)
+    return raw, decode(raw)
+
+
+def check_hash(raw, expected):
+    if not isinstance(expected, str) or not re.fullmatch('[0-9a-f]{64}', expected) or digest(raw) != expected:
+        raise ValueError('SHA256 mismatch')
+
+
+def identifier(scheme, value):
+    if not isinstance(value, str) or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value):
+        raise ValueError('Invalid identifier')
+    if scheme == 'arxiv':
+        pattern = r'(?:\d{2}(?:0[1-9]|1[0-2])\.\d{4,5}|[a-zA-Z][a-zA-Z.-]*/\d{2}(?:0[1-9]|1[0-2])\d{3})'
+        valid = re.fullmatch(pattern, value)
+    elif scheme == 'doi':
+        valid = re.fullmatch(r'10\.\d{4,9}/\S+', value) and value == value.lower()
+    else:
+        valid = False
+    if not valid:
+        raise ValueError('Unsupported or noncanonical identifier: ' + scheme)
+
+
+def prepare(plan_path, output):
+    plan_raw, plan = metadata(plan_path)
+    object_keys(plan, ('format_version', 'inputs', 'assertions'))
+    if type(plan['format_version']) is not int or plan['format_version'] != 1:
+        raise ValueError('Unsupported format version')
+    if not isinstance(plan['inputs'], list) or not plan['inputs'] or not isinstance(plan['assertions'], list) or not plan['assertions']:
+        raise ValueError('Inputs and assertions must be nonempty lists')
+    output = Path(output).absolute()
+    if output.exists():
+        raise ValueError('Output directory must not already exist')
+    if not output.parent.is_dir():
+        raise ValueError('Output parent directory must exist')
+    assertions = {}
+    evidence_files = {}
+    checked_evidence = {}
+    for assertion in plan['assertions']:
+        object_keys(assertion, ('source', 'external_id', 'record_sha256', 'identifiers', 'evidence'))
+        if any(not isinstance(assertion[key], str) or not assertion[key] for key in ('source', 'external_id')):
+            raise ValueError('Invalid source identity')
+        key = (assertion['source'], assertion['external_id'])
+        if key in assertions:
+            raise ValueError('Duplicate identity assertion')
+        ids = assertion['identifiers']
+        if not isinstance(ids, dict) or not ids:
+            raise ValueError('Identifiers must be a nonempty object')
+        for scheme, value in ids.items():
+            identifier(scheme, value)
+        if not isinstance(assertion['evidence'], list) or not assertion['evidence']:
+            raise ValueError('Evidence is required')
+        references = []
+        for evidence in assertion['evidence']:
+            object_keys(evidence, ('file', 'sha256', 'url', 'checked_at', 'note'))
+            if not all(isinstance(value, str) and value for value in evidence.values()):
+                raise ValueError('Invalid evidence fields')
+            url = urlsplit(evidence['url'])
+            if url.scheme != 'https' or not url.hostname or url.username or url.password:
+                raise ValueError('Evidence requires an HTTPS URL without credentials')
+            if datetime.fromisoformat(evidence['checked_at'].replace('Z', '+00:00')).utcoffset() is None:
+                raise ValueError('Evidence timestamp requires a timezone')
+            evidence_key = (str(Path(evidence['file']).absolute()), evidence['sha256'])
+            if evidence_key not in checked_evidence:
+                checked_evidence[evidence_key] = metadata(evidence['file'], evidence['sha256'])[0]
+            raw = checked_evidence[evidence_key]
+            evidence_files[evidence['sha256']] = raw
+            references.append({**evidence, 'file': 'evidence/' + evidence['sha256'] + '.json'})
+        assertions[key] = (assertion, references)
+    seen = set()
+    applied = set()
+    names = set()
+    report = {'format_version': 1, 'plan_sha256': digest(plan_raw),
+              'verification_scope': SCOPE, 'semantic_identity_verified': False,
+              'database_written': False, 'files': [], 'assertions_applied': 0, 'identifiers_added': 0}
+    with tempfile.TemporaryDirectory(prefix='.identifier-curation-', dir=output.parent) as temporary:
+        staging = Path(temporary) / 'prepared'
+        staging.mkdir()
+        (staging / 'evidence').mkdir()
+        (staging / 'plan.json').write_bytes(plan_raw)
+        for sha, raw in evidence_files.items():
+            (staging / 'evidence' / (sha + '.json')).write_bytes(raw)
+        for entry in plan['inputs']:
+            object_keys(entry, ('file', 'sha256', 'manifest_sha256'))
+            path = Path(entry['file'])
+            if path.suffix != '.jsonl' or path.name in names:
+                raise ValueError('Inputs require distinct JSONL basenames')
+            names.add(path.name)
+            _, manifest = metadata(path.with_suffix('.manifest.json'), entry['manifest_sha256'])
+            if not isinstance(manifest, dict) or not {'source','venue','year','track','count','list_url','file_sha256'} <= manifest.keys():
+                raise ValueError('Complete source manifest required')
+            if manifest['file_sha256'] != entry['sha256']:
+                raise ValueError('Manifest and plan hashes differ')
+            before = hashlib.sha256()
+            after = hashlib.sha256()
+            count = 0
+            with path.open('rb') as source, (staging / path.name).open('wb') as target:
+                for raw in source:
+                    before.update(raw)
+                    row = decode(raw)
+                    if any(row[field] != manifest[field] for field in ('source', 'venue', 'year', 'track')):
+                        raise ValueError('Record differs from source manifest')
+                    key = (row['source'], row['external_id'])
+                    if key in seen:
+                        raise ValueError('Duplicate source identity in input batch')
+                    seen.add(key)
+                    if key in assertions:
+                        assertion, references = assertions[key]
+                        check_hash(raw, assertion['record_sha256'])
+                        existing = row.get('identifiers', {})
+                        if existing is None:
+                            existing = {}
+                        if not isinstance(existing, dict):
+                            raise ValueError('Invalid existing identifiers')
+                        for scheme, value in assertion['identifiers'].items():
+                            if scheme in existing and existing[scheme] != value:
+                                raise ValueError('Existing identifier conflict')
+                        details = row.setdefault('metadata', {})
+                        if not isinstance(details, dict):
+                            raise ValueError('Invalid record metadata')
+                        history = details.setdefault('identifier_curation', [])
+                        if not isinstance(history, list):
+                            raise ValueError('Invalid curation history')
+                        history.append({'plan_sha256': digest(plan_raw), 'input_file_sha256': entry['sha256'],
+                                        'record_sha256': digest(raw), 'previous_identifiers': dict(existing),
+                                        'identifiers_asserted': assertion['identifiers'], 'evidence': references,
+                                        'verification_scope': SCOPE})
+                        report['identifiers_added'] += len(assertion['identifiers'].keys() - existing.keys())
+                        row['identifiers'] = {**existing, **assertion['identifiers']}
+                        raw = json_bytes(row)
+                        applied.add(key)
+                    target.write(raw)
+                    after.update(raw)
+                    count += 1
+            if before.hexdigest() != entry['sha256'] or not re.fullmatch('[0-9a-f]{64}', entry['sha256']):
+                raise ValueError('Input SHA256 mismatch')
+            if not count or type(manifest['count']) is not int or count != manifest['count']:
+                raise ValueError('Manifest count mismatch')
+            provenance = {'plan_sha256': digest(plan_raw), 'input_file_sha256': before.hexdigest(),
+                          'input_manifest_sha256': entry['manifest_sha256'], 'verification_scope': SCOPE}
+            prepared = {**manifest, 'file': str(output / path.name), 'file_sha256': after.hexdigest(),
+                        'identifier_curation': provenance}
+            (staging / path.with_suffix('.manifest.json').name).write_bytes(json_bytes(prepared))
+            report['files'].append({'file': str(output / path.name), 'count': count,
+                                    'input_sha256': before.hexdigest(), 'sha256': after.hexdigest()})
+        if applied != assertions.keys():
+            raise ValueError('Assertion target absent from input batch')
+        report['assertions_applied'] = len(applied)
+        (staging / 'curation-report.json').write_bytes(json_bytes(report))
+        # All files are on the same filesystem. No database or input mutation.
+        if output.exists():
+            raise ValueError('Output directory appeared during preparation')
+        staging.rename(output)
+    return report
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog, description=__doc__)
+    parser.add_argument('--plan', required=True, type=Path)
+    parser.add_argument('--output', required=True, type=Path)
+    args = parser.parse_args(argv)
+    print(json.dumps(prepare(args.plan, args.output), ensure_ascii=False))

paper_atlas/db/annotations.sql

@@ -0,0 +1,15 @@
+CREATE TABLE annotation_events (
+ id INTEGER PRIMARY KEY,
+ request_id TEXT NOT NULL UNIQUE,
+ paper_id INTEGER NOT NULL REFERENCES papers(id),
+ original_paper_id INTEGER NOT NULL,
+ kind TEXT NOT NULL CHECK(kind IN ('tag','research_type')),
+ namespace TEXT NOT NULL, value TEXT NOT NULL,
+ source_record_id INTEGER REFERENCES source_records(id),
+ evidence_key TEXT NOT NULL, evidence_url TEXT,
+ reason TEXT NOT NULL, actor TEXT NOT NULL, observed_at TEXT NOT NULL,
+ retracts_id INTEGER UNIQUE REFERENCES annotation_events(id),
+ CHECK(retracts_id IS NULL OR retracts_id<>id)
+);
+CREATE INDEX annotation_events_paper ON annotation_events(paper_id,kind,id);
+CREATE INDEX annotation_events_label ON annotation_events(kind,namespace,value,paper_id);

paper_atlas/db/edition_scopes.py

@@ -0,0 +1,38 @@
+"""Retain reviewed event scopes when a source refresh repeats a whole-volume label."""
+import json
+
+KEY = 'edition_scope_curation_v1'
+
+
+def preserve(metadata, previous):
+    old = json.loads(previous).get(KEY) if previous else None
+    if KEY in metadata and metadata[KEY] != old:
+        raise ValueError('Source import cannot supply or replace reviewed edition scope')
+    if old is not None:
+        metadata[KEY] = old
+    return metadata
+
+
+def resolve(c, record, external, venue, year, track, event_key):
+    row = c.execute('SELECT metadata_json FROM source_records WHERE id=?', (record,)).fetchone()
+    entries = json.loads(row[0]).get(KEY, []) if row else []
+    if not isinstance(entries, list):
+        raise ValueError('Invalid reviewed edition scope')
+    applicable = []
+    for entry in entries:
+        if entry['external_id'] != external:
+            continue
+        old, target = entry['from'], entry['target']
+        if int(year) != old['year'] or venue not in entry['venue_names']:
+            continue
+        if track not in (old['track'], target['track']):
+            continue
+        if event_key is not None and event_key != old['event_key']:
+            continue
+        current = c.execute('SELECT venue_id,year,track,event_key FROM editions WHERE id=?', (target['edition_id'],)).fetchone()
+        if not current or [current[i] for i in range(4)] != [target[k] for k in ('venue_id','year','track','event_key')]:
+            raise ValueError('Reviewed target edition has changed')
+        applicable.append(target['edition_id'])
+    if len(applicable) > 1:
+        raise ValueError('Ambiguous reviewed edition scope')
+    return applicable[0] if applicable else None

paper_atlas/db/events.py

@@ -0,0 +1,71 @@
+"""Stable event identity within a venue/year/track, independent of paper identity."""
+import json
+import re
+
+
+def has_event_key(c):
+    if getattr(c, 'postgres', False):
+        return bool(c.execute("SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='editions' AND column_name='event_key'").fetchone())
+    return any(row[1] == 'event_key' for row in c.execute('PRAGMA table_info(editions)'))
+
+
+def validate_event_key(value):
+    if value is not None and (not isinstance(value, str) or
+            (value != '' and not re.fullmatch(r'[a-z0-9][a-z0-9._:-]{0,79}', value))):
+        raise ValueError('Invalid conference event_key')
+    return value
+
+
+def pinned_www_fall(source, metadata):
+    from paper_atlas.collectors.dblp import XML_SHA
+    b = metadata.get('bibliography') or {}
+    urls = b.get('url') or []
+    return (source == 'dblp' and metadata.get('snapshot_sha256') == XML_SHA
+            and metadata.get('bibliographic_year') == 1995
+            and str(metadata.get('proceedings_key', '')).startswith('conf/www/')
+            and b.get('booktitle') == ['WWW (Fall)'] and len(urls) == 1
+            and urls[0].split('#')[0] == 'db/conf/www/www1995f.html')
+
+
+def find_edition(store, venue_id, venue, year, track, record, external_id,
+                 event_key=None, *, create=True):
+    c = store.conn
+    validate_event_key(event_key)
+    enabled = has_event_key(c) if getattr(store, 'event_identity', None) is None else store.event_identity
+    known_multi = venue.upper() == 'WWW' and int(year) in (1994, 1995)
+    source = c.execute('SELECT src.name,r.metadata_json FROM source_records r JOIN sources src ON src.id=r.source_id WHERE r.id=?', (record,)).fetchone()
+    inferred = ('www-1995-fall' if known_multi and int(year) == 1995 and source
+                and pinned_www_fall(source[0], json.loads(source[1])) else None)
+    if inferred:
+        if event_key is not None and event_key != inferred:
+            raise ValueError('event_key conflicts with pinned WWW Fall source evidence')
+        event_key = inferred
+    if not enabled:
+        if event_key or known_multi:
+            raise ValueError('Explicit event identity migration required before this conference import')
+        row = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?', (venue_id, year, track)).fetchone()
+        if row or not create:
+            return row[0] if row else None
+        c.execute('INSERT INTO editions(venue_id,year,track) VALUES(?,?,?) ON CONFLICT DO NOTHING', (venue_id, year, track))
+        return c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?', (venue_id, year, track)).fetchone()[0]
+    anchored = c.execute('''SELECT DISTINCT e.id,e.event_key FROM editions e JOIN submissions s ON s.edition_id=e.id
+        JOIN submission_sources ss ON ss.submission_id=s.id WHERE e.venue_id=? AND e.year=? AND e.track=?
+        AND ss.source_record_id=? AND ss.source_submission_id=?''', (venue_id, year, track, record, external_id)).fetchall()
+    if len(anchored) > 1:
+        raise ValueError('Source record already has ambiguous conference events')
+    if anchored:
+        if event_key is not None and event_key != anchored[0][1]:
+            raise ValueError('Cannot move an existing source to another conference event')
+        event_key = anchored[0][1]
+    if event_key is None:
+        rows = c.execute('SELECT event_key FROM editions WHERE venue_id=? AND year=? AND track=?', (venue_id, year, track)).fetchall()
+        if len(rows) > 1 or any(row[0] for row in rows):
+            raise ValueError('Explicit event_key required; conference event is ambiguous')
+        event_key = ''
+    if known_multi and not event_key:
+        raise ValueError('Explicit event_key required for same-year WWW events')
+    row = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?', (venue_id, year, track, event_key)).fetchone()
+    if row or not create:
+        return row[0] if row else None
+    c.execute('INSERT INTO editions(venue_id,year,track,event_key) VALUES(?,?,?,?) ON CONFLICT DO NOTHING', (venue_id, year, track, event_key))
+    return c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?', (venue_id, year, track, event_key)).fetchone()[0]

paper_atlas/db/schema.sql

@@ -29,7 +29,8 @@ CREATE TABLE IF NOT EXISTS venues (
 CREATE TABLE IF NOT EXISTS editions (
  id INTEGER PRIMARY KEY, venue_id INTEGER NOT NULL REFERENCES venues(id),
  year INTEGER NOT NULL, track TEXT NOT NULL DEFAULT 'main',
- UNIQUE(venue_id,year,track)
+ event_key TEXT NOT NULL DEFAULT '',
+ UNIQUE(venue_id,year,track,event_key)
 );
 CREATE TABLE IF NOT EXISTS venue_aliases (
  venue_id INTEGER NOT NULL REFERENCES venues(id), alias TEXT NOT NULL,

paper_atlas/db/store.py

@@ -36,6 +36,8 @@ class Store:
         self.conn.execute('PRAGMA synchronous=NORMAL')
         self.conn.execute('PRAGMA cache_size=-32768')
         migrate_sources(self.conn)
+        from paper_atlas.db.events import has_event_key
+        self.event_identity = has_event_key(self.conn)
 
     def source(self,name,url):
         self.conn.execute('INSERT INTO sources(name,homepage) VALUES(?,?) ON CONFLICT DO NOTHING',(name,url))
@@ -55,6 +57,9 @@ class Store:
         c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other))
         c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other))
         c.execute('UPDATE submissions SET paper_id=? WHERE paper_id=?',(target,other))
+        from paper_atlas.curation.annotations import available
+        if available(c):
+            c.execute('UPDATE annotation_events SET paper_id=? WHERE paper_id=?', (target,other))
         c.execute('INSERT INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=? ON CONFLICT DO NOTHING',(target,other))
         c.execute('DELETE FROM paper_tags WHERE paper_id=?',(other,))
         c.execute('UPDATE paper_merges SET surviving_paper_id=? WHERE surviving_paper_id=?',(target,other))
@@ -91,17 +96,25 @@ class Store:
             if found:
                 matches.add(found[0])
         old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
+        from paper_atlas.db.edition_scopes import preserve
+        metadata = preserve(dict(metadata or {}), old[4] if old else None)
         if old:
             matches.add(old[0])
         key = normalized(title)
         title_matches = c.execute('SELECT id,authors_json,abstract FROM papers WHERE title_key=?',(key,)).fetchall()
-        authors_set = author_keys(authors)
-        for paper,old_authors,old_abstract in title_matches:
-            overlap = authors_set & author_keys(json.loads(old_authors))
-            # Cross-platform title matching needs independent author corroboration.
-            same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone()
-            if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())):
-                matches.add(paper)
+        # An existing identifier/source record anchors the entity. Title
+        # similarity must not pull in another deposit or an extended version.
+        if not matches:
+            title_only_matches = set()
+            authors_set = author_keys(authors)
+            for paper,old_authors,old_abstract in title_matches:
+                overlap = authors_set & author_keys(json.loads(old_authors))
+                same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone()
+                if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())):
+                    title_only_matches.add(paper)
+            if len(title_only_matches)>1:
+                raise ValueError('Ambiguous paper identity: multiple author-corroborated title matches; provide verified identifiers')
+            matches = title_only_matches
         if matches:
             paper = min(matches)
             for other in sorted(matches-{paper}):
@@ -142,15 +155,32 @@ class Store:
             tag=c.execute('SELECT id FROM tags WHERE namespace=? AND value=?',(namespace,value)).fetchone()[0]
             c.execute('INSERT INTO paper_tags VALUES(?,?,?,?) ON CONFLICT DO NOTHING',(paper,tag,record,evidence))
 
-    def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None,identities=None,observed_at=None,decision_date=None,source_decision_id='',submitted_at=None,guard_status=False):
+    def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None,identities=None,observed_at=None,decision_date=None,source_decision_id='',submitted_at=None,guard_status=False,event_key=None):
         c = self.conn
+        # An already collected file may still call these exact pinned volumes
+        # IJCAI. Resolve membership before finding/creating an edition so a
+        # later refresh cannot split the repaired joint event again.
+        if venue == 'IJCAI' and year in (2018,2022) and track == 'main':
+            from paper_atlas.collectors.dblp import XML_SHA
+            evidence=c.execute('SELECT src.name,r.metadata_json FROM source_records r JOIN sources src ON src.id=r.source_id WHERE r.id=?',(record,)).fetchone()
+            if evidence and evidence[0] == 'dblp':
+                metadata=json.loads(evidence[1])
+                if (metadata.get('snapshot_sha256') == XML_SHA
+                        and metadata.get('proceedings_key') == 'conf/ijcai/'+str(year)
+                        and metadata.get('bibliographic_year') == year
+                        and (metadata.get('bibliography') or {}).get('booktitle') in (['IJCAI'],['IJCAI/ECAI'])):
+                    venue='ECAI+IJCAI'
+                    aliases=sorted(set(aliases or []) | {'ECAI','IJCAI'})
         existing=c.execute('SELECT id FROM venues WHERE lower(acronym)=lower(?)',(venue,)).fetchone()
         if existing:venue_id=existing[0]
         else:venue_id=inserted_id(c,"INSERT INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified'))
         for alias in aliases or []:
             c.execute('INSERT INTO venue_aliases VALUES(?,?) ON CONFLICT DO NOTHING',(venue_id,alias))
-        c.execute('INSERT INTO editions(venue_id,year,track) VALUES(?,?,?) ON CONFLICT DO NOTHING',(venue_id,int(year),track))
-        edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0]
+        from paper_atlas.db.events import find_edition
+        from paper_atlas.db.edition_scopes import resolve
+        edition = resolve(c,record,external_id,venue,year,track,event_key)
+        if edition is None:
+            edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key)
         # A previous event in this row may already have merged the input paper.
         paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0]
         submission=attach_submission(self,edition,paper,record,external_id,identities or [])

paper_atlas/db/submissions.py

@@ -4,7 +4,7 @@ import re
 from paper_atlas.io.publication_ids import mlsys_id
 
 
-PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp'}
+PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings'}
 
 
 def timestamp(value=None):

paper_atlas/ingestion/backfill.py

@@ -37,9 +37,14 @@ def backfill(db,files, *, pg_service=None, pg_schema=None):
                     seen.add(row['external_id'])
                     if any(row.get(key)!=manifest.get(key) for key in ('source','venue','year','track')):
                         raise ValueError('Evidence row differs from edition manifest')
+                    if 'event_key' in manifest and row.get('event_key')!=manifest['event_key']:
+                        raise ValueError('Event identity differs from manifest')
                     require_track_record(row, manifest)
                     record=c.execute('SELECT r.id,r.paper_id,r.metadata_json FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?',(row['source'],row['external_id'])).fetchone()
-                    edition=c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE lower(v.acronym)=lower(?) AND e.year=? AND e.track=?',(row['venue'],row['year'],row['track'])).fetchone()
+                    from paper_atlas.db.events import find_edition
+                    venue=c.execute('SELECT id FROM venues WHERE lower(acronym)=lower(?)',(row['venue'],)).fetchone()
+                    eid=find_edition(store,venue[0],row['venue'],row['year'],row['track'],record[0],row['external_id'],row.get('event_key'),create=False) if record and venue else None
+                    edition=(eid,) if eid is not None else None
                     if not record or not edition:raise ValueError('Evidence backfill requires an already imported record and edition')
                     if not c.execute('SELECT 1 FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id WHERE ss.source_record_id=? AND ss.source_submission_id=? AND s.edition_id=?',(record[0],row['external_id'],edition[0])).fetchone():
                         raise ValueError('No existing submission to backfill')

paper_atlas/ingestion/edition_tracks.py

@@ -0,0 +1,245 @@
+"""Snapshot-bound reclassification of existing attempts, without reimport or merge."""
+import argparse
+from collections import Counter
+from contextlib import closing
+import hashlib
+import json
+from pathlib import Path
+import re
+
+from paper_atlas.db.backend import database_arguments, open_read_connection, open_store, inserted_id
+from paper_atlas.db.edition_scopes import KEY
+from paper_atlas.ingestion.event_identity import database_identity, digest, values
+
+MARKER = 'edition_tracks_v1:'
+
+
+def validate(inventory):
+    if (not isinstance(inventory, dict) or inventory.get('format') != 1
+            or not isinstance(inventory.get('records'), list) or not 0 < len(inventory['records']) <= 10000):
+        raise ValueError('Nonempty bounded reviewed inventory required')
+    ids = set(); records = set()
+    for row in inventory['records']:
+        required = {'submission_id','source_record_id','external_id','source','doi','from_edition_id',
+                    'year','venue','venue_names','track','section','evidence_url','evidence_sha256'}
+        if not isinstance(row, dict) or set(row) != required:
+            raise ValueError('Unexpected inventory record fields')
+        for field in ('submission_id','source_record_id','from_edition_id','year'):
+            if type(row[field]) is not int or row[field] <= 0:
+                raise ValueError('Positive integer inventory identity required')
+        if row['submission_id'] in ids or row['source_record_id'] in records:
+            raise ValueError('Repeated submission/source identity in inventory')
+        ids.add(row['submission_id']); records.add(row['source_record_id'])
+        for field in ('external_id','source','doi','venue','section','evidence_url'):
+            if not isinstance(row[field], str) or not row[field].strip():
+                raise ValueError('Missing inventory evidence field')
+        if (not isinstance(row['venue_names'], list) or not row['venue_names']
+                or not all(isinstance(v,str) and v for v in row['venue_names'])
+                or row['venue'] not in row['venue_names']
+                or not re.fullmatch(r'[a-z][a-z0-9_]{0,63}', row['track'])
+                or not re.fullmatch(r'[0-9a-f]{64}', row['evidence_sha256'])
+                or not row['evidence_url'].startswith('https://')):
+            raise ValueError('Invalid scope or evidence binding')
+    return inventory
+
+
+def bounded(c, sql, params):
+    rows = values(c.execute(sql+' LIMIT 100001', params))
+    if len(rows) > 100000:
+        raise ValueError('Protected scope exceeds bounded reclassification limit')
+    return rows
+
+
+def protect(c, ids):
+    marks = ','.join('?' for _ in ids)
+    subs = 'SELECT id FROM submissions WHERE id IN ('+marks+')'
+    papers = 'SELECT paper_id FROM submissions WHERE id IN ('+marks+')'
+    decisions = 'SELECT id FROM decisions WHERE submission_id IN ('+subs+')'
+    queries = {
+        'submissions': ('SELECT id,paper_id,source_record_id,source_submission_id,submitted_at FROM submissions WHERE id IN ('+marks+') ORDER BY id'),
+        'submission_identifiers': ('SELECT scheme,value,submission_id FROM submission_identifiers WHERE submission_id IN ('+subs+') ORDER BY submission_id,scheme,value'),
+        'submission_sources': ('SELECT * FROM submission_sources WHERE submission_id IN ('+subs+') ORDER BY submission_id,source_record_id,source_submission_id'),
+        'decisions': ('SELECT * FROM decisions WHERE submission_id IN ('+subs+') ORDER BY id'),
+        'decision_sources': ('SELECT * FROM decision_sources WHERE decision_id IN ('+decisions+') ORDER BY decision_id,source_record_id,source_decision_id'),
+        'papers': ('SELECT * FROM papers WHERE id IN ('+papers+') ORDER BY id'),
+        'identifiers': ('SELECT * FROM identifiers WHERE paper_id IN ('+papers+') ORDER BY scheme,value'),
+        'paper_tags': ('SELECT * FROM paper_tags WHERE paper_id IN ('+papers+') ORDER BY paper_id,tag_id,evidence_key'),
+    }
+    from paper_atlas.curation.annotations import available
+    if available(c):
+        queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN ('+papers+') ORDER BY id'
+    result = {table: {'rows':len(rows), 'sha256':digest(rows)} for table, sql in queries.items()
+              for rows in [bounded(c,sql,ids)]}
+    rows = bounded(c, 'SELECT r.* FROM source_records r WHERE r.id IN (SELECT source_record_id FROM submission_sources WHERE submission_id IN ('+subs+')) ORDER BY r.id', ids)
+    # Only the reviewed curation field may change; every other metadata value
+    # and all raw text/IDs/retrieval times remain protected.
+    for row in rows:
+        metadata = json.loads(row[8]); metadata.pop(KEY,None); row[8] = metadata
+    result['source_records'] = {'rows':len(rows), 'sha256':digest(rows)}
+    return result
+
+
+def inspect(c, inventory):
+    validate(inventory); changes=[]; ids=tuple(sorted(r['submission_id']for r in inventory['records']))
+    original = sorted({r['from_edition_id'] for r in inventory['records']})
+    actual_ids = {r[0] for r in c.execute('SELECT id FROM submissions WHERE edition_id IN ('+','.join('?'for _ in original)+')', original)}
+    if actual_ids != set(ids):
+        raise ValueError('Inventory must cover every attempt in each original edition')
+    for row in inventory['records']:
+        current = c.execute('''SELECT s.edition_id,s.source_record_id,s.source_submission_id,s.paper_id,
+            e.venue_id,e.year,e.track,e.event_key,v.acronym,r.external_id,r.metadata_json,src.name
+            FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
+            JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
+        if not current or [current[i]for i in (0,1,2,5,8,9,11)] != [row[k]for k in ('from_edition_id','source_record_id','external_id','year','venue','external_id','source')]:
+            raise ValueError('Submission identity/scope differs from inventory')
+        aliases={r[0]for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?',(current[4],))}|{current[8]}
+        if set(row['venue_names'])!=aliases:
+            raise ValueError('Venue names differ from current aliases')
+        metadata=json.loads(current[10])
+        if KEY in metadata:
+            raise ValueError('Existing curated scope requires explicit reconciliation')
+        if (metadata.get('record_identifiers') or {}).get('doi') != row['doi']:
+            raise ValueError('Stable DOI differs from reviewed source evidence')
+        links=values(c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(row['submission_id'],)))
+        if links != [[row['source_record_id'],row['external_id']]]:
+            raise ValueError('Shared-source attempt requires separate reconciliation')
+        decisions=values(c.execute('SELECT status FROM decisions WHERE submission_id=?',(row['submission_id'],)))
+        if not decisions or any(d[0]!='accepted'for d in decisions):
+            raise ValueError('Only publication observations can use this track correction')
+        target=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',(current[4],current[5],row['track'],current[7])).fetchone()
+        if target and target[0] != current[0]:
+            if c.execute('SELECT 1 FROM submissions WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
+                raise ValueError('Nonempty target edition requires separate reconciliation')
+            if c.execute('SELECT 1 FROM submission_identifiers WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
+                raise ValueError('Target identifier scope already exists')
+            if c.execute('SELECT 1 FROM edition_sources WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
+                raise ValueError('Target coverage already exists; reconcile first')
+        if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],current[0])).fetchone():
+            raise ValueError('Existing identifier edition is inconsistent')
+        changes.append({**row,'from_track':current[6],'event_key':current[7], 'venue_id':current[4],
+                        'target_edition_id':target[0]if target else None})
+    coverage=bounded(c,'SELECT * FROM edition_sources WHERE edition_id IN ('+','.join('?'for _ in original)+') ORDER BY edition_id,source_id,list_url',original)
+    if Counter(row[0]for row in coverage)!=Counter({edition:1 for edition in original}):
+        raise ValueError('Exactly one original coverage listing per edition required')
+    return {'records':changes, 'protected':protect(c,ids), 'coverage':coverage,
+            'editions':bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)}
+
+
+def proposal(c, inventory, identity):
+    return {'format':1,'database':identity,'inventory':inventory,'before':inspect(c,inventory)}
+
+
+def verify(c, plan):
+    records=plan['before']['records']; ids=tuple(sorted(r['submission_id']for r in records))
+    if protect(c,ids)!=plan['before']['protected']:
+        raise ValueError('Protected history/text/tags/identities changed; roll back')
+    for row in records:
+        event=c.execute('''SELECT e.id,e.venue_id,e.year,e.track,e.event_key FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
+        if not event or [event[i]for i in range(1,5)] != [row['venue_id'],row['year'],row['track'],row['event_key']]:
+            raise ValueError('Target track assignment failed')
+        if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],event[0])).fetchone():
+            raise ValueError('Target identifier assignment failed')
+        metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0])
+        if row['track'] != row['from_track']:
+            if metadata.get(KEY) != [curation(row,event[0],digest(plan))]:
+                raise ValueError('Reviewed refresh protection missing or changed')
+        elif KEY in metadata:
+            raise ValueError('Unexpected scope revision on unchanged track')
+    original=sorted({r['from_edition_id']for r in records})
+    if bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)!=plan['before']['editions']:
+        raise ValueError('Original edition identities changed')
+    for old in plan['before']['coverage']:
+        edition,source,url,_,_,_,cursor,success,_=old
+        expected=Counter()
+        for row in records:
+            if row['from_edition_id']==edition:
+                target=c.execute('SELECT edition_id FROM submissions WHERE id=?',(row['submission_id'],)).fetchone()[0]
+                expected[target]+=1
+        for target,number in expected.items():
+            actual=c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources WHERE edition_id=? AND source_id=? AND list_url=?',(target,source,url)).fetchone()
+            note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+digest(plan)
+            if actual is None or [actual[i]for i in range(6)]!=['partial',number,number,cursor,success,note]:
+                raise ValueError('Corrected coverage differs from reviewed scope')
+    return {'records_verified':len(records),'protected':plan['before']['protected']}
+
+
+def curation(row, target, signature):
+    return {'external_id':row['external_id'], 'venue_names':sorted(set(row['venue_names'])),
+            'from':{'edition_id':row['from_edition_id'],'year':row['year'],'track':row['from_track'],'event_key':row['event_key']},
+            'target':{'edition_id':target,'venue_id':row['venue_id'],'year':row['year'],'track':row['track'],'event_key':row['event_key']},
+            'section':row['section'],'evidence_url':row['evidence_url'],'evidence_sha256':row['evidence_sha256'],'plan_digest':signature}
+
+
+def apply_plan(c, plan):
+    if plan.get('format') != 1:
+        raise ValueError('Unsupported plan format')
+    signature=digest(plan);marker=MARKER+signature
+    if c.execute('SELECT 1 FROM schema_migrations WHERE name=?',(marker,)).fetchone():
+        return {'status':'already_applied','events_moved':0,**verify(c,plan)}
+    if inspect(c,plan['inventory']) != plan['before']:
+        raise ValueError('Stale track plan; create and review a fresh plan')
+    targets={}; counts=Counter(); moved=0
+    for row in plan['before']['records']:
+        key=(row['venue_id'],row['year'],row['track'],row['event_key'])
+        if key not in targets:
+            found=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',key).fetchone()
+            targets[key]=found[0]if found else inserted_id(c,'INSERT INTO editions(venue_id,year,track,event_key) VALUES(?,?,?,?)',key)
+        target=targets[key];counts[(row['from_edition_id'],target)]+=1
+        if target != row['from_edition_id']:
+            c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?',(target,row['submission_id']))
+            if c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',(target,row['submission_id'],row['from_edition_id'])).rowcount!=1:
+                raise ValueError('Concurrent submission change')
+            metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0]);metadata[KEY]=[curation(row,target,signature)]
+            c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),row['source_record_id']));moved+=1
+    # Preserve each original source URL and timestamps, but do not retain its
+    # old whole-volume count as main-track coverage. Partial is deliberate:
+    # this operation verifies scope, not abstracts or all possible sources.
+    for old in plan['before']['coverage']:
+        edition,source,url,status,expected,imported,cursor,success,error=old
+        own={r['submission_id'] for r in plan['before']['records']if r['from_edition_id']==edition}
+        source_name=c.execute('SELECT name FROM sources WHERE id=?',(source,)).fetchone()[0]
+        if {r['source']for r in plan['before']['records']if r['submission_id']in own}!={source_name}:
+            raise ValueError('Coverage includes a source outside the reviewed inventory')
+        for (origin,target),number in counts.items():
+            if origin!=edition:continue
+            note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+signature
+            c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,cursor,last_success_at,last_error)
+                VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
+                expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_error=excluded.last_error''',
+                (target,source,url,'partial',number,number,cursor,success,note))
+    result=verify(c,plan)
+    c.execute('INSERT INTO schema_migrations(name) VALUES(?)',(marker,))
+    return {'status':'applied','events_moved':moved,**result}
+
+
+def run(db,out,*,pg_service=None,pg_schema=None,inventory=None,plan=None,plan_sha256=None):
+    out=Path(out)
+    if out.exists() or not out.parent.is_dir():raise ValueError('New report path in existing directory required')
+    expected=None
+    if plan:
+        raw=Path(plan).read_bytes()
+        if hashlib.sha256(raw).hexdigest()!=plan_sha256:raise ValueError('Plan hash differs')
+        expected=json.loads(raw);validate(expected['inventory'])
+    elif plan_sha256:raise ValueError('Plan path required')
+    if bool(expected)==bool(inventory):raise ValueError('Choose inventory preflight or apply plan')
+    if expected is None:
+        with closing(open_read_connection(db,pg_service=pg_service,pg_schema=pg_schema))as c:
+            result=proposal(c,validate(json.loads(Path(inventory).read_bytes())),database_identity(c,db))
+    else:
+        store=open_store(db,pg_service=pg_service,pg_schema=pg_schema)
+        with closing(store.conn)as c:
+            try:
+                if expected['database']!=database_identity(c,db):raise ValueError('Plan belongs to another database')
+                if not getattr(c,'postgres',False):c.commit();c.execute('BEGIN IMMEDIATE')
+                result=apply_plan(c,expected);c.commit()
+            except BaseException:c.rollback();raise
+    out.write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');return result
+
+
+def main(argv=None,*,prog=None):
+    p=argparse.ArgumentParser(prog=prog);database_arguments(p)
+    p.add_argument('--inventory');p.add_argument('--plan');p.add_argument('--plan-sha256');p.add_argument('--out',required=True)
+    a=p.parse_args(argv)
+    if bool(a.plan)!=bool(a.plan_sha256):p.error('--plan and --plan-sha256 must be supplied together')
+    r=run(a.db,a.out,pg_service=a.pg_service,pg_schema=a.pg_schema,inventory=a.inventory,plan=a.plan,plan_sha256=a.plan_sha256)
+    print(json.dumps({'status':r.get('status','planned'),'events_moved':r.get('events_moved',0),'out':a.out}),flush=True)

paper_atlas/ingestion/event_identity.py

@@ -0,0 +1,221 @@
+"""Explicit, plan-bound edition event-key upgrade; never reimport submissions."""
+import argparse
+from contextlib import closing
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+import sys
+
+from paper_atlas.db.backend import database_arguments, open_read_connection, require_sqlite_writable
+from paper_atlas.db.events import has_event_key, pinned_www_fall
+from paper_atlas.postgres.schema import TABLES, OPTIONAL_TABLES, quote
+
+MARKER = 'edition_event_identity_v1:'
+
+
+def digest(value):
+    return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
+
+
+def values(rows):
+    return [[r[i] for i in range(len(r))] for r in rows]
+
+
+def exact_counts(c, *, page_size=5000, progress=None):
+    """Count every row in the same snapshot through bounded primary-key pages."""
+    if not isinstance(page_size, int) or page_size < 1:
+        raise ValueError('Positive count page size required')
+    result = {}
+    if getattr(c, 'postgres', False):
+        optional = [table for table in OPTIONAL_TABLES if c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (table,)).fetchone()]
+    else:
+        optional = [table for table in OPTIONAL_TABLES if c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()]
+    for table in (*TABLES, *optional):
+        if table == 'schema_migrations':
+            continue
+        if getattr(c, 'postgres', False):
+            columns = [r[0] for r in c.execute("SELECT a.attname FROM pg_constraint pc JOIN pg_attribute a ON a.attrelid=pc.conrelid AND a.attnum=ANY(pc.conkey) WHERE pc.conrelid=?::regclass AND pc.contype='p' ORDER BY array_position(pc.conkey,a.attnum)", (table,))]
+        else:
+            columns = [r[1] for r in sorted(c.execute('PRAGMA table_info('+quote(table)+')'), key=lambda r:r[5]) if r[5]]
+        if not columns:
+            raise ValueError('Primary key required for exact count: '+table)
+        fields = ','.join(quote(column) for column in columns)
+        last = None; count = 0; pages = 0
+        if progress:
+            progress({'table':table, 'rows_counted':0, 'complete':False})
+        while True:
+            where = (' WHERE ('+fields+') > ('+','.join('?' for _ in columns)+')') if last is not None else ''
+            batch = values(c.execute('SELECT '+fields+' FROM '+quote(table)+where+' ORDER BY '+fields+' LIMIT ?', (*last, page_size) if last is not None else (page_size,)).fetchall())
+            count += len(batch); pages += 1
+            complete = len(batch) < page_size
+            if progress and (complete or pages % 20 == 0):
+                progress({'table':table, 'rows_counted':count, 'complete':complete})
+            if complete:
+                break
+            last = batch[-1]
+        result[table] = count
+    return result
+
+
+def inspect(c, *, progress=None):
+    enabled = has_event_key(c)
+    if getattr(c, 'postgres', False):
+        columns = [r[0] for r in c.execute("SELECT column_name FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='editions' ORDER BY ordinal_position")]
+        unique = [r[0] for r in c.execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='editions'::regclass AND contype='u'")]
+        expected_unique = ['UNIQUE (venue_id, year, track, event_key)' if enabled else 'UNIQUE (venue_id, year, track)']
+    else:
+        columns = [r[1] for r in c.execute('PRAGMA table_info(editions)')]
+        indexes = list(c.execute('PRAGMA index_list(editions)'))
+        unique = [tuple(r[2] for r in c.execute('PRAGMA index_info('+quote(index[1])+')')) for index in indexes if index[2]]
+        expected_unique = [('venue_id','year','track','event_key') if enabled else ('venue_id','year','track')]
+        if len(indexes) != 1:
+            raise ValueError('Unexpected edition indexes; require a reviewed migration')
+    if columns != ['id','venue_id','year','track'] + (['event_key'] if enabled else []) or unique != expected_unique:
+        raise ValueError('Unexpected edition schema; require a reviewed migration')
+    rows = values(c.execute('SELECT id,venue_id,year,track FROM editions ORDER BY id'))
+    keys = values(c.execute('SELECT id,event_key FROM editions ORDER BY id')) if enabled else [[r[0], ''] for r in rows]
+    counts = exact_counts(c, progress=progress)
+    targets = values(c.execute("SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym='WWW' AND e.year=1995 AND e.track='main' ORDER BY e.id"))
+    assignments = []
+    for (eid,) in targets:
+        current_key = dict(keys)[eid]
+        if current_key:
+            continue
+        evidence = values(c.execute('''SELECT s.id,src.name,r.id,r.external_id,r.metadata_json FROM submissions s
+            JOIN submission_sources ss ON ss.submission_id=s.id JOIN source_records r ON r.id=ss.source_record_id
+            JOIN sources src ON src.id=r.source_id WHERE s.edition_id=? ORDER BY s.id,r.id''', (eid,)))
+        count = c.execute('SELECT count(*) FROM submissions WHERE edition_id=?', (eid,)).fetchone()[0]
+        supported = {r[0] for r in evidence if pinned_www_fall(r[1], json.loads(r[4]))}
+        if not count or len(supported) != count:
+            raise ValueError('Existing unlabelled WWW1995 edition lacks complete pinned Fall evidence')
+        assignments.append({'edition_id':eid, 'event_key':'www-1995-fall', 'submissions':count,
+                            'source_evidence_sha256':digest(evidence)})
+    # Protect all rows in the affected attempt history, including non-DBLP evidence.
+    target_ids = [r[0] for r in targets]
+    protected = {}
+    for table,sql in {
+        'submissions': 'SELECT s.* FROM submissions s WHERE s.edition_id IN ({}) ORDER BY s.id',
+        'decisions': 'SELECT d.* FROM decisions d JOIN submissions s ON s.id=d.submission_id WHERE s.edition_id IN ({}) ORDER BY d.id',
+        'submission_sources': 'SELECT ss.* FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id WHERE s.edition_id IN ({}) ORDER BY ss.submission_id,ss.source_record_id,ss.source_submission_id',
+        'decision_sources': 'SELECT ds.* FROM decision_sources ds JOIN decisions d ON d.id=ds.decision_id JOIN submissions s ON s.id=d.submission_id WHERE s.edition_id IN ({}) ORDER BY ds.decision_id,ds.source_record_id,ds.source_decision_id',
+        'source_records': 'SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id JOIN submissions s ON s.id=ss.submission_id WHERE s.edition_id IN ({}) ORDER BY r.id',
+    }.items():
+        records = values(c.execute(sql.format(','.join('?' for _ in target_ids)), target_ids)) if target_ids else []
+        protected[table] = {'count':len(records), 'sha256':digest(records)}
+    return {'event_column':enabled, 'editions_sha256':digest(rows), 'edition_ids':[r[0] for r in rows],
+            'event_keys':keys, 'counts':counts, 'assignments':assignments, 'protected':protected}
+
+
+def database_identity(c, db=None):
+    if getattr(c, 'postgres', False):
+        r = c.execute('SELECT current_database(),current_schema()').fetchone()
+        return {'engine':'postgres', 'database':r[0], 'schema':r[1]}
+    return {'engine':'sqlite', 'path':str(Path(db).resolve())}
+
+
+def plan(db=None, *, pg_service=None, pg_schema=None, progress=None):
+    with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
+        return {'format':1, 'database':database_identity(c, db), 'before':inspect(c, progress=progress)}
+
+
+def apply_plan(c, proposal, *, progress=None):
+    before = proposal['before']
+    if proposal.get('format') != 1:
+        raise ValueError('Unsupported event identity plan')
+    if getattr(c, 'postgres', False):
+        c.execute('LOCK TABLE editions IN ACCESS EXCLUSIVE MODE')
+    current = inspect(c, progress=progress)
+    signature = digest(proposal)
+    marker = MARKER + signature
+    if c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (marker,)).fetchone():
+        expected_keys = dict(before['event_keys'])
+        expected_keys.update({a['edition_id']:a['event_key'] for a in before['assignments']})
+        if (not current['event_column'] or current['editions_sha256'] != before['editions_sha256']
+                or current['protected'] != before['protected'] or current['counts'] != before['counts']
+                or dict(current['event_keys']) != expected_keys):
+            raise ValueError('Applied plan no longer matches current data; generate a new plan')
+        return {'status':'already_applied', 'updated_editions':0, 'after':current}
+    if current != before:
+        raise ValueError('Event identity plan is stale; generate a new plan')
+    if not current['event_column']:
+        if getattr(c, 'postgres', False):
+            constraints = c.execute("SELECT conname,pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='editions'::regclass AND contype='u'").fetchall()
+            matches = [name for name,definition in constraints if definition == 'UNIQUE (venue_id, year, track)']
+            if len(matches) != 1 or len(constraints) != 1:
+                raise ValueError('Unexpected legacy edition uniqueness')
+            c.execute("ALTER TABLE editions ADD COLUMN event_key TEXT COLLATE \"C\" NOT NULL DEFAULT ''")
+            c.execute('ALTER TABLE editions DROP CONSTRAINT '+quote(matches[0]))
+            c.execute('ALTER TABLE editions ADD CONSTRAINT editions_event_identity_unique UNIQUE(venue_id,year,track,event_key)')
+        else:
+            unique = [tuple(r[2] for r in c.execute('PRAGMA index_info('+quote(index[1])+')'))
+                      for index in c.execute('PRAGMA index_list(editions)') if index[2]]
+            if unique != [('venue_id','year','track')]:
+                raise ValueError('Unexpected legacy edition uniqueness')
+            c.execute("CREATE TABLE editions_event_upgrade(id INTEGER PRIMARY KEY,venue_id INTEGER NOT NULL REFERENCES venues(id),year INTEGER NOT NULL,track TEXT NOT NULL DEFAULT 'main',event_key TEXT NOT NULL DEFAULT '',UNIQUE(venue_id,year,track,event_key))")
+            c.execute('INSERT INTO editions_event_upgrade(id,venue_id,year,track) SELECT id,venue_id,year,track FROM editions')
+            c.execute('DROP TABLE editions')
+            c.execute('ALTER TABLE editions_event_upgrade RENAME TO editions')
+    for item in before['assignments']:
+        result = c.execute("UPDATE editions SET event_key=? WHERE id=? AND event_key=''", (item['event_key'],item['edition_id']))
+        if result.rowcount != 1:
+            raise ValueError('Edition assignment changed during upgrade')
+    c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (marker,))
+    after = inspect(c, progress=progress)
+    expected_keys = dict(before['event_keys'])
+    expected_keys.update({a['edition_id']:a['event_key'] for a in before['assignments']})
+    if (after['editions_sha256'] != before['editions_sha256'] or after['protected'] != before['protected']
+            or after['counts'] != before['counts'] or dict(after['event_keys']) != expected_keys):
+        raise ValueError('Event upgrade changed protected data')
+    if not getattr(c, 'postgres', False) and c.execute('PRAGMA foreign_key_check').fetchall():
+        raise ValueError('Foreign key violation after event upgrade')
+    return {'status':'applied', 'updated_editions':len(before['assignments']), 'after':after}
+
+
+def upgrade(proposal, db=None, *, pg_service=None, pg_schema=None, progress=None):
+    if pg_service:
+        from paper_atlas.postgres.writer import WriteConnection
+        with closing(WriteConnection(pg_service, pg_schema)) as c:
+            if proposal['database'] != database_identity(c):
+                raise ValueError('Plan targets a different database')
+            try:
+                result = apply_plan(c, proposal, progress=progress); c.commit(); return result
+            except BaseException:
+                c.rollback(); raise
+    require_sqlite_writable(db)
+    if not Path(db).is_file():
+        raise ValueError('Existing SQLite sample required')
+    with closing(sqlite3.connect(db, timeout=30)) as c:
+        if proposal['database'] != database_identity(c, db):
+            raise ValueError('Plan targets a different database')
+        c.execute('PRAGMA foreign_keys=OFF')  # Outside the transaction; no cascading deletes.
+        try:
+            c.execute('BEGIN IMMEDIATE')
+            result = apply_plan(c, proposal, progress=progress); c.commit(); return result
+        except BaseException:
+            c.rollback(); raise
+        finally:
+            c.execute('PRAGMA foreign_keys=ON')
+
+
+def main(argv=None, *, prog=None):
+    p = argparse.ArgumentParser(prog=prog); database_arguments(p)
+    p.add_argument('--out', required=True)
+    p.add_argument('--apply-plan'); p.add_argument('--plan-sha256')
+    args = p.parse_args(argv)
+    if bool(args.apply_plan) != bool(args.plan_sha256):
+        p.error('--apply-plan and --plan-sha256 must be supplied together')
+    target = Path(args.out)
+    if target.exists():
+        raise ValueError('Refusing to overwrite an existing plan/report')
+    progress = lambda event: print(json.dumps({'phase':'exact_row_count', **event}), file=sys.stderr, flush=True)
+    if args.apply_plan:
+        raw = Path(args.apply_plan).read_bytes()
+        if hashlib.sha256(raw).hexdigest() != args.plan_sha256:
+            raise ValueError('Plan file SHA256 differs from approved bytes')
+        result = upgrade(json.loads(raw), args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, progress=progress)
+    else:
+        result = plan(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, progress=progress)
+    target.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
+    print(json.dumps({'out':str(target), 'sha256':hashlib.sha256(target.read_bytes()).hexdigest(),
+                      'status':result.get('status','planned')}), flush=True)

paper_atlas/ingestion/joint_editions.py

@@ -0,0 +1,204 @@
+"""Evidence-bound repair of two existing IJCAI/ECAI edition memberships."""
+import argparse
+from contextlib import closing
+import hashlib
+import json
+from pathlib import Path
+
+from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.collectors.dblp import XML_SHA
+
+YEARS = (2018, 2022)
+TARGET = 'ECAI+IJCAI'
+MEMBERS = {'ECAI', 'IJCAI'}
+EVIDENCE = 'https://www.eurai.org/ecai/history'
+MARKER = 'joint_ijcai_ecai_2018_2022_v1'
+
+
+def _digest(c, sql, params=()):
+    sha = hashlib.sha256(); count = 0
+    for row in c.execute(sql, params):
+        sha.update((json.dumps([row[i] for i in range(len(row))], ensure_ascii=False,
+                              separators=(',', ':')) + '\n').encode())
+        count += 1
+        if count > 100000:
+            raise ValueError('Joint repair evidence scope exceeded bounded row limit')
+    return {'rows': count, 'sha256': sha.hexdigest()}
+
+
+def fingerprints(c, ids):
+    marks = ','.join('?' for _ in ids)
+    submissions = 'SELECT id FROM submissions WHERE edition_id IN (' + marks + ')'
+    papers = 'SELECT paper_id FROM submissions WHERE edition_id IN (' + marks + ')'
+    decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + submissions + ')'
+    checks = {
+        'papers': ('id IN (' + papers + ')', 'id'),
+        'identifiers': ('paper_id IN (' + papers + ')', 'scheme,value'),
+        'source_records': ('paper_id IN (' + papers + ')', 'id'),
+        'paper_tags': ('paper_id IN (' + papers + ')', 'paper_id,tag_id,evidence_key'),
+        'submissions': ('edition_id IN (' + marks + ')', 'id'),
+        'decisions': ('submission_id IN (' + submissions + ')', 'id'),
+        'submission_sources': ('submission_id IN (' + submissions + ')', 'submission_id,source_record_id,source_submission_id'),
+        'submission_identifiers': ('edition_id IN (' + marks + ')', 'edition_id,scheme,value'),
+        'decision_sources': ('decision_id IN (' + decisions + ')', 'decision_id,source_record_id,source_decision_id'),
+        'edition_sources': ('edition_id IN (' + marks + ')', 'edition_id,source_id,list_url'),
+    }
+    return {table: _digest(c, 'SELECT * FROM ' + table + ' WHERE ' + where + ' ORDER BY ' + order, ids)
+            for table, (where, order) in checks.items()}
+
+
+def inspect(c):
+    target = c.execute('SELECT id FROM venues WHERE acronym=?', (TARGET,)).fetchall()
+    if len(target) != 1:
+        raise ValueError('Exactly one existing canonical joint venue required')
+    target = target[0][0]
+    aliases = {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (target,))}
+    if aliases != MEMBERS:
+        raise ValueError('Joint target must already have exactly ECAI and IJCAI aliases')
+    editions = []
+    for year in YEARS:
+        rows = c.execute('''SELECT e.id,e.venue_id,v.acronym FROM editions e JOIN venues v ON v.id=e.venue_id
+            WHERE v.acronym IN ('IJCAI','ECAI+IJCAI') AND e.year=? AND e.track='main' ''', (year,)).fetchall()
+        if len(rows) != 1:
+            raise ValueError('Expected one existing edition per year; missing or colliding editions require separate review')
+        edition, venue, acronym = (rows[0][i] for i in range(3))
+        count = c.execute('SELECT count(*) FROM submissions WHERE edition_id=?', (edition,)).fetchone()[0]
+        if not 0 < count <= 5000:
+            raise ValueError('Existing edition must have a nonempty bounded submission scope')
+        # Every existing submission must be supported by its own pinned DBLP
+        # volume record. No title/author matching, new attempts or text writes.
+        evidence = c.execute('''SELECT s.id,r.external_id,r.metadata_json FROM submissions s
+            JOIN submission_sources ss ON ss.submission_id=s.id
+            JOIN source_records r ON r.id=ss.source_record_id JOIN sources src ON src.id=r.source_id
+            WHERE s.edition_id=? AND src.name='dblp' ORDER BY s.id,r.id''', (edition,)).fetchall()
+        seen = set()
+        for row in evidence:
+            sid, external, metadata = (row[i] for i in range(3))
+            meta = json.loads(metadata)
+            bibliography = meta.get('bibliography') or {}
+            if (meta.get('snapshot_sha256') != XML_SHA
+                    or meta.get('proceedings_key') != 'conf/ijcai/' + str(year)
+                    or meta.get('bibliographic_year') != year
+                    or bibliography.get('booktitle') not in (['IJCAI'], ['IJCAI/ECAI'])
+                    or not external.startswith('conf/ijcai/')):
+                raise ValueError('Source evidence outside the exact verified IJCAI/ECAI volume')
+            seen.add(sid)
+        if len(seen) != count:
+            raise ValueError('Every submission must have matching original volume evidence')
+        editions.append({'id': edition, 'year': year, 'track': 'main', 'from_venue_id': venue,
+                         'from_venue': acronym, 'target_venue_id': target, 'submissions': count,
+                         'evidence_url': EVIDENCE, 'proceedings_key': 'conf/ijcai/' + str(year)})
+    ids = tuple(e['id'] for e in editions)
+    return {'format_version': 1, 'kind': MARKER, 'checked_at': timestamp(), 'editions': editions,
+            'content_fingerprints': fingerprints(c, ids), 'target_aliases': sorted(aliases),
+            'status': 'already_joint' if all(e['from_venue_id'] == target for e in editions) else 'ready',
+            'scope': 'Only these two existing edition venue references; no source text, paper, submission, decision, track, identifier or tag change.'}
+
+
+def validate_plan(expected):
+    if not isinstance(expected, dict):
+        raise ValueError('Repair plan must be a JSON object')
+    if set(expected) != {'format_version','kind','checked_at','editions','content_fingerprints','target_aliases','status','scope','database'}:
+        raise ValueError('Unexpected repair plan fields')
+    if (type(expected['format_version']) is not int or expected['format_version'] != 1
+            or expected['kind'] != MARKER or expected['status'] not in ('ready','already_joint')):
+        raise ValueError('Unsupported repair plan')
+    if (not isinstance(expected['database'],dict) or not isinstance(expected['content_fingerprints'],dict)
+            or not isinstance(expected['editions'],list) or len(expected['editions']) != 2
+            or expected['target_aliases'] != sorted(MEMBERS)):
+        raise ValueError('Invalid repair plan structure')
+    for edition, year in zip(expected['editions'], YEARS):
+        if not isinstance(edition,dict) or edition.get('year') != year:
+            raise ValueError('Repair plan must name the two exact ordered years')
+        for key in ('id','year','from_venue_id','target_venue_id','submissions'):
+            if type(edition.get(key)) is not int or edition[key] <= 0:
+                raise ValueError('Repair plan IDs/counts must be positive integers')
+
+
+def apply_plan(c, expected):
+    validate_plan(expected)
+    actual = inspect(c)
+    if actual['content_fingerprints'] != expected['content_fingerprints'] or actual['target_aliases'] != expected['target_aliases']:
+        raise ValueError('Stale repair plan: protected content or aliases changed')
+    if len(expected['editions']) != 2:
+        raise ValueError('Exactly two editions required')
+    for old, now in zip(expected['editions'], actual['editions']):
+        if set(old) != set(now):
+            raise ValueError('Unexpected edition fields')
+        for key in ('id','year','track','target_venue_id','submissions','evidence_url','proceedings_key'):
+            if old[key] != now[key]:
+                raise ValueError('Stale repair plan: edition differs')
+        if now['from_venue_id'] != old['from_venue_id'] and now['from_venue_id'] != now['target_venue_id']:
+            raise ValueError('Edition membership differs from plan')
+    if actual['status'] == 'already_joint':
+        return {**actual, 'status':'already_joint', 'updated_editions':0}
+    for edition in actual['editions']:
+        if edition['from_venue_id'] == edition['target_venue_id']:
+            continue
+        changed = c.execute('''UPDATE editions SET venue_id=? WHERE id=? AND venue_id=? AND year=? AND track=?''',
+                            (edition['target_venue_id'],edition['id'],edition['from_venue_id'],edition['year'],edition['track']))
+        if changed.rowcount != 1:
+            raise ValueError('Concurrent edition change; entire repair must roll back')
+    after = inspect(c)
+    if after['status'] != 'already_joint' or after['content_fingerprints'] != actual['content_fingerprints']:
+        raise ValueError('Postcheck changed protected content; entire repair must roll back')
+    c.execute('INSERT INTO schema_migrations(name) VALUES(?) ON CONFLICT DO NOTHING', (MARKER,))
+    return {**after, 'status':'applied', 'updated_editions':sum(e['from_venue_id'] != e['target_venue_id'] for e in actual['editions'])}
+
+
+def database_identity(c, db):
+    if getattr(c, 'postgres', False):
+        row = c.execute('SELECT current_database(),current_schema()').fetchone()
+        return {'engine':'postgresql','database':row[0],'schema':row[1]}
+    return {'engine':'sqlite','path':str(Path(db).resolve())}
+
+
+def run(db, out, *, pg_service=None, pg_schema=None, plan=None, plan_sha256=None):
+    out = Path(out).resolve()
+    if not out.parent.is_dir():
+        raise ValueError('Report parent directory must already exist')
+    if out.exists():
+        raise ValueError('Preserve existing evidence report; choose a new output path')
+    expected = None
+    if plan:
+        raw = Path(plan).read_bytes()
+        if hashlib.sha256(raw).hexdigest() != plan_sha256:
+            raise ValueError('Repair plan hash differs before opening database')
+        expected = json.loads(raw)
+        validate_plan(expected)
+    elif plan_sha256:
+        raise ValueError('Plan hash requires an existing plan')
+    if db and out == Path(db).resolve():
+        raise ValueError('Report must not overwrite database')
+    writer = open_store(db, pg_service=pg_service, pg_schema=pg_schema) if expected is not None else None
+    c = writer.conn if writer else open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)
+    with closing(c):
+        try:
+            if expected is not None and not getattr(c, 'postgres', False):
+                c.commit(); c.execute('BEGIN IMMEDIATE')
+            identity = database_identity(c, db)
+            if expected is not None and expected['database'] != identity:
+                raise ValueError('Plan belongs to another database/schema')
+            report = apply_plan(c, expected) if expected is not None else inspect(c)
+            report['database'] = identity
+            if expected is not None:
+                report['plan_sha256'] = plan_sha256
+            if expected is not None:
+                c.commit()
+        except BaseException:
+            if expected is not None: c.rollback()
+            raise
+    out.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
+    print(json.dumps({'status':report['status'],'editions':len(report['editions']),
+                      'updated_editions':report.get('updated_editions',0),'report':str(out)}))
+    return report
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
+    parser.add_argument('--out', required=True)
+    parser.add_argument('--plan', help='Apply an already reviewed plan; omitted means read-only preflight')
+    parser.add_argument('--plan-sha256', help='Required exact plan SHA256 when applying')
+    args = parser.parse_args(argv)
+    run(args.db,args.out,pg_service=args.pg_service,pg_schema=args.pg_schema,plan=args.plan,plan_sha256=args.plan_sha256)

paper_atlas/ingestion/records.py

@@ -12,6 +12,69 @@ from paper_atlas.collectors.public_openreview import SOURCE as PUBLIC_SOURCE
 from paper_atlas.collectors.icml_public_archive import SOURCE as ARCHIVE_SOURCE
 
 
+def coverage_options(manifest):
+    mode = manifest.get('coverage_mode', 'snapshot')
+    if mode not in ('snapshot', 'incremental'):
+        raise ValueError('Unsupported coverage mode')
+    if mode == 'snapshot':
+        return None
+    required = ('venue', 'year', 'track', 'count', 'coverage_previous_count',
+                'coverage_previous_expected_count', 'coverage_expected_count')
+    if any(key not in manifest for key in required) or manifest.get('status') != 'partial':
+        raise ValueError('Incremental coverage requires a partial, explicitly scoped manifest')
+    previous = manifest['coverage_previous_count']
+    expected = manifest['coverage_expected_count']
+    prior_expected = manifest['coverage_previous_expected_count']
+    if (type(previous) is not int or previous < 0 or type(expected) is not int or expected < 1
+            or type(manifest['count']) is not int or manifest['count'] < 1
+            or expected < max(previous, manifest['count'])
+            or (prior_expected is not None and (type(prior_expected) is not int or prior_expected < 0
+                                               or expected < prior_expected))):
+        raise ValueError('Invalid incremental coverage counts')
+    return previous, prior_expected, expected
+
+
+def source_record_count(c, source, edition):
+    return c.execute('''SELECT count(DISTINCT ss.source_record_id) FROM submission_sources ss
+        JOIN submissions s ON s.id=ss.submission_id
+        JOIN source_records sr ON sr.id=ss.source_record_id
+        WHERE s.edition_id=? AND sr.source_id=?''', (edition, source)).fetchone()[0]
+
+
+def require_incremental_parent(c, source, manifest, options):
+    from paper_atlas.db.events import has_event_key, validate_event_key
+    key = validate_event_key(manifest.get('event_key'))
+    enabled = has_event_key(c)
+    if key and not enabled:
+        raise ValueError('Event identity migration required for incremental coverage')
+    field = 'e.event_key' if enabled else "''"
+    rows = c.execute('SELECT e.id,'+field+""" FROM editions e JOIN venues v ON v.id=e.venue_id
+        WHERE lower(v.acronym)=lower(?) AND e.year=? AND e.track=?""",
+        (manifest['venue'], manifest['year'], manifest['track'])).fetchall()
+    if key is None:
+        if len(rows) > 1 or any(r[1] for r in rows) or (manifest['venue'] == 'WWW' and manifest['year'] in (1994,1995)):
+            raise ValueError('Explicit manifest event_key required for incremental coverage')
+    else:
+        rows = [r for r in rows if r[1] == key]
+    if len(rows) > 1:
+        raise ValueError('Ambiguous incremental coverage event')
+    edition = rows[0] if rows else None
+    entries = c.execute('''SELECT list_url,status,expected_count,imported_count
+        FROM edition_sources WHERE edition_id=? AND source_id=?''',
+        (edition[0], source)).fetchall() if edition else []
+    previous, prior_expected, _ = options
+    if entries:
+        if len(entries) != 1 or entries[0][0] != manifest['list_url']:
+            raise ValueError('Incremental coverage requires one unambiguous source listing')
+        if tuple(entries[0][1:]) != ('partial', prior_expected, previous):
+            raise ValueError('Incremental coverage parent changed or is not partial')
+    elif previous != 0 or prior_expected is not None:
+        raise ValueError('Incremental coverage parent is absent')
+    actual = source_record_count(c, source, edition[0]) if edition else 0
+    if actual != previous:
+        raise ValueError('Incremental coverage parent differs from actual source records')
+
+
 def ingest_file(db,path, *, pg_service=None, pg_schema=None):
     path = Path(path)
     manifest_path = path.with_suffix('.manifest.json')
@@ -24,6 +87,7 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
     source_name = manifest.get('source')
     if not source_name:
         raise ValueError('A source manifest is required')
+    incremental = coverage_options(manifest)
     store = open_store(db, pg_service=pg_service, pg_schema=pg_schema)
     c = store.conn
     run = None
@@ -35,6 +99,8 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
         source_id = store.source(source_name,manifest['list_url'])
         run = inserted_id(c,'INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path),'expected_sha256':manifest.get('file_sha256')})))
         c.commit()
+        if incremental is not None:
+            require_incremental_parent(c, source_id, manifest, incremental)
         digest = hashlib.sha256()
         with path.open('rb') as stream:
             for line in stream:
@@ -42,9 +108,11 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
                 row = json.loads(line)
                 if row['source'] != source_name:
                     raise ValueError('Source differs from manifest')
-                if any(key in manifest and manifest[key]!=row[key] for key in ('venue','year','track')):
+                if any(key in manifest and manifest[key]!=row.get(key) for key in ('venue','year','track','event_key')):
                     raise ValueError('Conference edition differs from manifest')
                 require_track_record(row, manifest)
+                from paper_atlas.db.events import validate_event_key
+                validate_event_key(row.get('event_key'))
                 external = row['external_id']
                 if external in seen:
                     raise ValueError('Duplicate external ID in one collected file: '+external)
@@ -65,7 +133,13 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
                         status=event['status']
                         if events and status is None:raise ValueError('Explicit decision event requires a status')
                         if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event')
-                        edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'),guard_status=source_name in (PUBLIC_SOURCE,ARCHIVE_SOURCE))
+                        edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'),guard_status=source_name in (PUBLIC_SOURCE,ARCHIVE_SOURCE),event_key=row.get('event_key'))
+                    from paper_atlas.db.edition_scopes import KEY
+                    scope_metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])
+                    if scope_metadata.get(KEY):
+                        actual_track=c.execute('SELECT track FROM editions WHERE id=?',(edition,)).fetchone()[0]
+                        if row['track']!=actual_track:
+                            raise ValueError('Recollect this whole-volume source with reviewed per-record tracks before refresh')
                     editions.add(edition)
                 else:
                     raise ValueError('This importer requires an explicit supported decision status')
@@ -82,8 +156,25 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
         if expected is not None and expected != count:
             raise ValueError(f'Manifest count {expected} differs from records {count}')
         complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0) and not missing_abstracts
-        c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
-        c.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),count,count,json.dumps({'file':str(path),'file_sha256':digest.hexdigest()}),run))
+        imported = count
+        condition = ''
+        condition_params = ()
+        if incremental is not None:
+            previous, prior_expected, expected = incremental
+            imported = source_record_count(c, source_id, edition)
+            if imported < previous or imported > expected:
+                raise ValueError('Incremental coverage exceeds the declared source bounds')
+            complete = False
+            condition = " WHERE edition_sources.imported_count=? AND COALESCE(edition_sources.expected_count,-1)=? AND edition_sources.status='partial'"
+            condition_params = (previous, -1 if prior_expected is None else prior_expected)
+        result = c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL'+condition,(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,imported,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),*condition_params))
+        if incremental is not None and result.rowcount != 1:
+            raise ValueError('Incremental coverage parent changed during import')
+        checkpoint = {'file':str(path),'file_sha256':digest.hexdigest()}
+        if incremental is not None:
+            checkpoint['coverage'] = {'mode':'incremental','previous_count':previous,
+                                      'expected_count':expected,'imported_count':imported}
+        c.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),count,count,json.dumps(checkpoint),run))
         c.commit()
         print(json.dumps({'file':str(path),'status':'imported','records':count,'missing_abstract_count':missing_abstracts}),flush=True)
     except BaseException as exc:

paper_atlas/postgres/migration.py

@@ -143,7 +143,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
     free_space(storage_path, min_free_gib)
     control = schema + '_migration'
     report = {'status': 'preflight', 'source': str(db), 'service': service, 'schema': schema,
-              'started_at': timestamp(), 'tables': {}, 'scope': 'All 21 business tables; FTS5 internals rebuilt as native PostgreSQL GIN.'}
+              'started_at': timestamp(), 'tables': {}, 'scope': 'All baseline business tables and recognized installed extensions; FTS5 rebuilt as native PostgreSQL GIN.'}
     with Path(str(db) + '.maintenance.lock').open('a') as lock:
         fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
         if Path(str(db) + '-wal').exists() and Path(str(db) + '-wal').stat().st_size:
@@ -240,7 +240,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
                     JOIN pg_am am ON am.oid=ix.relam WHERE ns.nspname=%s AND ix.relname='papers_search_gin' ''', (stage,)).fetchone()
                 if search != ('gin', True, True, 'papers'):
                     raise ValueError('Native full-text index is not valid and ready')
-                for table in TABLES:
+                for table in plan:
                     pg.execute('ANALYZE ' + qualify(stage, table))
                 invalid = pg.execute('''SELECT count(*) FROM pg_constraint co JOIN pg_class cl ON cl.oid=co.conrelid
                     JOIN pg_namespace ns ON ns.oid=cl.relnamespace WHERE ns.nspname=%s AND NOT co.convalidated''', (stage,)).fetchone()[0]

paper_atlas/postgres/schema.py

@@ -8,7 +8,9 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi
           'submission_sources', 'submission_identifiers', 'decision_sources',
           'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
           'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
+OPTIONAL_TABLES = ('annotation_events',)
 CHECKS = {
+    'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
     'venues': ["venue_type IN ('conference','journal','review_platform')"],
     'edition_sources': ["status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')"],
     'decisions': ["status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')"],
@@ -50,10 +52,10 @@ def quote(name):
 def inspect(connection):
     business = {name for schema, name, kind, *_ in connection.execute('PRAGMA table_list')
                 if schema == 'main' and kind == 'table' and not name.startswith('sqlite_')}
-    if business != set(TABLES):
+    if not set(TABLES) <= business or business - set(TABLES) - set(OPTIONAL_TABLES):
         raise ValueError('Unexpected business tables: ' + repr(sorted(business ^ set(TABLES))))
     result = {}
-    for table in TABLES:
+    for table in (*TABLES, *(name for name in OPTIONAL_TABLES if name in business)):
         columns = [{'name': r[1], 'type': r[2].upper(), 'notnull': bool(r[3]),
                     'default': r[4], 'pk': r[5]}
                    for r in connection.execute('PRAGMA table_info(' + quote(table) + ')')]

paper_atlas/postgres/writer.py

@@ -85,3 +85,5 @@ class WriteConnection:
 class PostgresStore(Store):
     def __init__(self, service, schema=None):
         self.conn = WriteConnection(service, schema)
+        from paper_atlas.db.events import has_event_key
+        self.event_identity = has_event_key(self.conn)

paper_atlas/querying.py

@@ -14,7 +14,9 @@ ACCEPTED_PAPERS = "SELECT s.paper_id FROM submissions s JOIN decisions d ON d.su
 
 
 def known_acceptances(conn,paper):
-    return [dict(row) for row in conn.execute('''SELECT s.id AS submission_id,v.acronym AS venue,e.year,e.track,
+    from paper_atlas.db.events import has_event_key
+    event = 'e.event_key' if has_event_key(conn) else "''"
+    return [dict(row) for row in conn.execute('''SELECT s.id AS submission_id,v.acronym AS venue,e.year,e.track,'''+event+''' AS event_key,
         d.id AS decision_id,d.decision_date,d.observed_at,d.evidence_url
         FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
         JOIN decisions d ON d.submission_id=s.id WHERE s.paper_id=? AND d.status='accepted' AND '''+LATEST_DECISION+
@@ -29,10 +31,13 @@ def main(argv=None, *, prog=None):
     parser.add_argument('--pg-schema', help='Optional PostgreSQL schema override; otherwise use the configured search_path')
     parser.add_argument('--query', help='Title/abstract search: FTS5 for SQLite, web-search syntax for PostgreSQL')
     parser.add_argument('--paper',type=int,help='Paper ID; include sources, tags and submission history')
+    parser.add_argument('--research-type', help='Independent type filter as namespace=value; no type inferred from track')
+    parser.add_argument('--annotation-history', action='store_true', help='Include retraction evidence for --paper')
     parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI')
     parser.add_argument('--venue', help='Conference acronym')
     parser.add_argument('--year', type=int)
     parser.add_argument('--track',help='Exact track, e.g. main, findings, workshop')
+    parser.add_argument('--event-key',help='Exact event within a venue/year/track')
     parser.add_argument('--status', choices=['accepted','rejected','desk_rejected','withdrawn','under_review','unknown'])
     parser.add_argument('--outcome',choices=['accepted','no_known_acceptance'],help='跨会议的已知录用情况；未查到录用不等于从未录用')
     parser.add_argument('--limit', type=int, default=20)
@@ -47,6 +52,13 @@ def main(argv=None, *, prog=None):
         conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True)
         conn.row_factory = sqlite3.Row
         conn.execute('BEGIN')  # One consistent read snapshot while ingestion continues.
+    from paper_atlas.db.events import has_event_key
+    event = 'e.event_key' if has_event_key(conn) else "''"
+    if args.event_key is not None and event == "''":
+        conn.close()
+        raise ValueError('Event identity migration required for event-key queries')
+    from paper_atlas.curation.annotations import available, observations, ACTIVE
+    has_annotations = available(conn)
     if args.stats:
         result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')}
         result['records_by_source'] = [dict(r) for r in conn.execute('SELECT s.name,count(r.id) AS imported_records FROM sources s LEFT JOIN source_records r ON r.source_id=s.id GROUP BY s.id ORDER BY s.name')]
@@ -69,17 +81,35 @@ def main(argv=None, *, prog=None):
                 where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)')
             params.append(args.query)
         if args.tag:
+            if '=' not in args.tag:
+                parser.error('--tag requires namespace=value')
             namespace,value = args.tag.split('=',1)
-            where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value])
+            platform = 'EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'
+            params.extend([namespace,value])
+            if has_annotations:
+                platform = '(' + platform + ' OR EXISTS(SELECT 1 FROM annotation_events a WHERE a.paper_id=p.id AND a.kind=\'tag\' AND a.namespace=? AND a.value=? AND ' + ACTIVE + '))'
+                params.extend([namespace,value])
+            where.append(platform)
+        if args.research_type:
+            if '=' not in args.research_type:
+                parser.error('--research-type requires namespace=value')
+            namespace,value = args.research_type.split('=',1)
+            if has_annotations:
+                where.append('EXISTS(SELECT 1 FROM annotation_events a WHERE a.paper_id=p.id AND a.kind=\'research_type\' AND a.namespace=? AND a.value=? AND ' + ACTIVE + ')')
+                params.extend([namespace,value])
+            else:
+                where.append('1=0')
         if args.outcome:
             where.append('p.id '+('IN' if args.outcome=='accepted' else 'NOT IN')+' ('+ACCEPTED_PAPERS+')')
-        if args.venue or args.year or args.status or args.track:
+        if args.venue or args.year or args.status or args.track or args.event_key is not None:
             clause = ['1=1']
             if args.venue:
                 if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='venue_aliases'").fetchone():
                     clause.append('(lower(v.acronym)=lower(?) OR EXISTS(SELECT 1 FROM venue_aliases va WHERE va.venue_id=v.id AND lower(va.alias)=lower(?)))'); params.extend([args.venue,args.venue])
                 else:
                     clause.append('lower(v.acronym)=lower(?)');params.append(args.venue)
+            if args.event_key is not None:
+                clause.append('e.event_key=?');params.append(args.event_key)
             if args.track:
                 clause.append('e.track=?');params.append(args.track)
             if args.year:
@@ -98,10 +128,14 @@ def main(argv=None, *, prog=None):
             paper['known_acceptances']=known_acceptances(conn,row['id'])
             paper['publication_outcome']='accepted' if paper['known_acceptances'] else 'no_known_acceptance'
             if args.paper is not None:
+                paper['research_types'] = observations(conn, row['id'], kind='research_type')
+                paper['annotation_tags'] = observations(conn, row['id'], kind='tag')
+                if args.annotation_history:
+                    paper['annotation_history'] = observations(conn, row['id'], history=True)
                 paper['tags'] = [dict(r) for r in conn.execute('SELECT t.namespace,t.value,pt.evidence_key FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=?',(row['id'],))]
                 paper['sources'] = [dict(r) for r in conn.execute('SELECT s.name,r.external_id,r.url,r.title,r.abstract,r.retrieved_at FROM source_records r JOIN sources s ON s.id=r.source_id WHERE r.paper_id=?',(row['id'],))]
                 submissions = []
-                for sub in conn.execute('SELECT s.id,s.source_submission_id,s.submitted_at,v.acronym,e.year,e.track FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year,s.id',(row['id'],)):
+                for sub in conn.execute('SELECT s.id,s.source_submission_id,s.submitted_at,v.acronym,e.year,e.track,'+event+' AS event_key FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year,s.id',(row['id'],)):
                     submission = dict(sub)
                     submission['decisions'] = []
                     for d in conn.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY observed_at,COALESCE(decision_date,observed_at),id',(sub['id'],)):

pyproject.toml

@@ -21,5 +21,5 @@ include = ["paper_atlas*"]
 exclude = ["tests*"]
 
 [tool.setuptools.package-data]
-"paper_atlas.db" = ["schema.sql"]
+"paper_atlas.db" = ["schema.sql", "annotations.sql"]
 "paper_atlas.resources" = ["venues.json"]

tests/fixtures/historical_predecessors.json

@@ -0,0 +1,314 @@
+{
+  "source_snapshot_sha256": "0376ad019b216553faff0a14e1757b2d4c7a3b84c1beacf0511ef83695211b18",
+  "scope": "Pinned proceedings metadata, not authentic example paper records or evidence that all volumes contain indexed papers. Test papers are synthetic.",
+  "volumes": [
+    {
+      "key": "conf/ecml/1987",
+      "fields": {
+        "editor": [
+          "Ivan Bratko",
+          "Nada Lavrac"
+        ],
+        "title": [
+          "Progress in Machine Learning - Proceedings of EWSL 87: 2nd European Working Session on Learning, Bled, Yugoslavia, May 1987"
+        ],
+        "booktitle": [
+          "EWSL"
+        ],
+        "publisher": [
+          "Sigma Press, Wilmslow"
+        ],
+        "year": [
+          "1987"
+        ],
+        "isbn": [
+          "1-85058-088-X"
+        ],
+        "url": [
+          "db/conf/ecml/ewsl87.html"
+        ]
+      },
+      "bibliographic_counts": [
+        {
+          "year": "1987",
+          "booktitle": "EWSL",
+          "record_type": "inproceedings",
+          "count": 16
+        }
+      ],
+      "expected_original_series": "EWSL",
+      "expected_target_family": "ECML-PKDD"
+    },
+    {
+      "key": "conf/ecml/1988",
+      "fields": {
+        "editor": [
+          "Derek H. Sleeman"
+        ],
+        "title": [
+          "Proceedings of the Third European Working Session on Learning, EWSL 1988, Turing Institute, Glasgow, UK, October 3-5, 1988"
+        ],
+        "booktitle": [
+          "EWSL"
+        ],
+        "publisher": [
+          "Pitman Publishing"
+        ],
+        "year": [
+          "1988"
+        ],
+        "isbn": [
+          "0-273-08800-9"
+        ],
+        "url": [
+          "db/conf/ecml/ewsl88.html"
+        ]
+      },
+      "bibliographic_counts": [
+        {
+          "year": "1988",
+          "booktitle": "EWSL",
+          "record_type": "inproceedings",
+          "count": 21
+        }
+      ],
+      "expected_original_series": "EWSL",
+      "expected_target_family": "ECML-PKDD"
+    },
+    {
+      "key": "conf/ecml/1989",
+      "fields": {
+        "editor": [
+          "Katharina Morik"
+        ],
+        "title": [
+          "Proceedings of the 4th European Working Session on Learning, Montpellier, France, 4-6 December 1989"
+        ],
+        "booktitle": [
+          "EWSL"
+        ],
+        "publisher": [
+          "Pitman and Morgan Kaufman Publishers"
+        ],
+        "year": [
+          "1989"
+        ]
+      },
+      "bibliographic_counts": [],
+      "expected_original_series": "EWSL",
+      "expected_target_family": "ECML-PKDD"
+    },
+    {
+      "key": "conf/ecml/1991",
+      "fields": {
+        "editor": [
+          "Yves Kodratoff"
+        ],
+        "title": [
+          "Machine Learning - EWSL-91, European Working Session on Learning, Porto, Portugal, March 6-8, 1991, Proceedings"
+        ],
+        "booktitle": [
+          "EWSL"
+        ],
+        "series": [
+          "Lecture Notes in Computer Science"
+        ],
+        "volume": [
+          "482"
+        ],
+        "publisher": [
+          "Springer"
+        ],
+        "year": [
+          "1991"
+        ],
+        "isbn": [
+          "3-540-53816-X"
+        ],
+        "ee": [
+          "https://doi.org/10.1007/BFb0016999"
+        ],
+        "url": [
+          "db/conf/ecml/ewsl91.html"
+        ]
+      },
+      "bibliographic_counts": [
+        {
+          "year": "1991",
+          "booktitle": "EWSL",
+          "record_type": "inproceedings",
+          "count": 44
+        }
+      ],
+      "expected_original_series": "EWSL",
+      "expected_target_family": "ECML-PKDD"
+    },
+    {
+      "key": "conf/ecp/1997",
+      "fields": {
+        "editor": [
+          "Sam Steel",
+          "Rachid Alami 0001"
+        ],
+        "title": [
+          "Recent Advances in AI Planning, 4th European Conference on Planning, ECP'97, Toulouse, France, September 24-26, 1997, Proceedings"
+        ],
+        "booktitle": [
+          "ECP"
+        ],
+        "series": [
+          "Lecture Notes in Computer Science"
+        ],
+        "volume": [
+          "1348"
+        ],
+        "publisher": [
+          "Springer"
+        ],
+        "year": [
+          "1997"
+        ],
+        "isbn": [
+          "3-540-63912-8"
+        ],
+        "ee": [
+          "https://doi.org/10.1007/3-540-63912-8"
+        ],
+        "url": [
+          "db/conf/ecp/ecp1997.html"
+        ]
+      },
+      "bibliographic_counts": [
+        {
+          "year": "1997",
+          "booktitle": "ECP",
+          "record_type": "inproceedings",
+          "count": 35
+        }
+      ],
+      "expected_original_series": "ECP",
+      "expected_target_family": "ICAPS"
+    },
+    {
+      "key": "conf/ecp/1999",
+      "fields": {
+        "editor": [
+          "Susanne Biundo",
+          "Maria Fox 0001"
+        ],
+        "title": [
+          "Recent Advances in AI Planning, 5th European Conference on Planning, ECP'99, Durham, UK, September 8-10, 1999, Proceedings"
+        ],
+        "booktitle": [
+          "ECP"
+        ],
+        "series": [
+          "Lecture Notes in Computer Science"
+        ],
+        "volume": [
+          "1809"
+        ],
+        "publisher": [
+          "Springer"
+        ],
+        "year": [
+          "2000"
+        ],
+        "isbn": [
+          "3-540-67866-2"
+        ],
+        "ee": [
+          "https://doi.org/10.1007/10720246"
+        ],
+        "url": [
+          "db/conf/ecp/ecp1999.html"
+        ]
+      },
+      "bibliographic_counts": [
+        {
+          "year": "1999",
+          "booktitle": "ECP",
+          "record_type": "inproceedings",
+          "count": 28
+        }
+      ],
+      "expected_original_series": "ECP",
+      "expected_target_family": "ICAPS"
+    },
+    {
+      "key": "conf/ewsp/1991",
+      "fields": {
+        "editor": [
+          "Joachim Hertzberg"
+        ],
+        "title": [
+          "European Workshop on Planning, EWSP '91, Sankt Augustin, Germany, March 18-19, 1991, Proceedings"
+        ],
+        "booktitle": [
+          "EWSP"
+        ],
+        "series": [
+          "Lecture Notes in Computer Science"
+        ],
+        "volume": [
+          "522"
+        ],
+        "publisher": [
+          "Springer"
+        ],
+        "year": [
+          "1991"
+        ],
+        "isbn": [
+          "3-540-54364-3"
+        ],
+        "ee": [
+          "https://doi.org/10.1007/BFb0052944"
+        ],
+        "url": [
+          "db/conf/ewsp/ewsp1991.html"
+        ]
+      },
+      "bibliographic_counts": [
+        {
+          "year": "1991",
+          "booktitle": "EWSP",
+          "record_type": "inproceedings",
+          "count": 10
+        }
+      ],
+      "expected_original_series": "EWSP",
+      "expected_target_family": "ICAPS"
+    },
+    {
+      "key": "conf/aips/1990",
+      "fields": {
+        "title": [
+          "Proceedings of the First International Conference on Expert Planning Systems, Brighton, UK, June 27-29, 1990"
+        ],
+        "booktitle": [
+          "Expert Planning Systems"
+        ],
+        "publisher": [
+          "AAAI"
+        ],
+        "year": [
+          "1990"
+        ],
+        "url": [
+          "db/conf/aips/aips1990.html"
+        ]
+      },
+      "bibliographic_counts": [
+        {
+          "year": "1990",
+          "booktitle": "Expert Planning Systems",
+          "record_type": "inproceedings",
+          "count": 48
+        }
+      ],
+      "expected_original_series": "EPS",
+      "expected_target_family": "ICAPS"
+    }
+  ]
+}

tests/fixtures/rss2025/p001.html

@@ -0,0 +1,148 @@
+<html> 
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />  
+<head>  
+<meta name="citation_title" content="Learned Perceptive Forward Dynamics Model for Safe and Platform-aware Robotic Navigation" />  
+<meta name="citation_author" content="Pascal Roth" />  
+<meta name="citation_author" content="Jonas Frey" />  
+<meta name="citation_author" content="Cesar Cadena" />  
+<meta name="citation_author" content="Marco Hutter" />  
+<meta name="citation_publication_date" content="2025/06/21" />  
+<meta name="citation_conference_title" content="Robotics: Science and Systems XXI" />  
+<meta name="citation_isbn" content="979-8-9902848-1-4" />  
+<meta name="citation_volume" content="21" />  
+<meta name="citation_pdf_url" content="http://www.roboticsproceedings.org/rss21/p001.pdf" />  
+<title>Robotics: Science and Systems XXI - Online Proceedings</title>  
+<link type="text/css" rel="stylesheet" href="../style.css">  
+<!--[if IE 5]> 
+<link type="text/css" rel="stylesheet" href="../ie5.css"> 
+<![endif]-->  
+<!--[if IE 6]> 
+<link type="text/css" rel="stylesheet" href="../ie6.css"> 
+<![endif]-->  
+</head>  
+<body>  
+<div class="menu">  
+<ul>
+<li class="here"><a class="menu" href="../index.html">Home</a></li>
+<li class="label">RSS XXII</span></li>
+<li class="menu"><a class="menu" href="../rss22/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss22/authors.html">Authors</a></li>
+<li class="label">RSS XXI</span></li>
+<li class="menu"><a class="menu" href="../rss21/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss21/authors.html">Authors</a></li>
+<li class="label">RSS XX</span></li>
+<li class="menu"><a class="menu" href="../rss20/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss20/authors.html">Authors</a></li>
+<li class="label">RSS XIX</span></li>
+<li class="menu"><a class="menu" href="../rss19/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss19/authors.html">Authors</a></li>
+<li class="label">RSS XVIII</span></li>
+<li class="menu"><a class="menu" href="../rss18/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss18/authors.html">Authors</a></li>
+<li class="label">RSS XVII</span></li>
+<li class="menu"><a class="menu" href="../rss17/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss17/authors.html">Authors</a></li>
+<li class="label">RSS XVI</span></li>
+<li class="menu"><a class="menu" href="../rss16/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss16/authors.html">Authors</a></li>
+<li class="label">RSS XV</span></li>
+<li class="menu"><a class="menu" href="../rss15/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss15/authors.html">Authors</a></li>
+<li class="label">RSS XIV</span></li>
+<li class="menu"><a class="menu" href="../rss14/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss14/authors.html">Authors</a></li>
+<li class="label">RSS XIII</span></li>
+<li class="menu"><a class="menu" href="../rss13/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss13/authors.html">Authors</a></li>
+<li class="label">RSS XII</span></li>
+<li class="menu"><a class="menu" href="../rss12/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss12/authors.html">Authors</a></li>
+<li class="label">RSS XI</span></li>
+<li class="menu"><a class="menu" href="../rss11/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss11/authors.html">Authors</a></li>
+<li class="label">RSS X</span></li>
+<li class="menu"><a class="menu" href="../rss10/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss10/authors.html">Authors</a></li>
+<li class="label">RSS IX</span></li>
+<li class="menu"><a class="menu" href="../rss09/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss09/authors.html">Authors</a></li>
+<li class="label">RSS VIII</span></li>
+<li class="menu"><a class="menu" href="../rss08/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss08/authors.html">Authors</a></li>
+<li class="label">RSS VII</span></li>
+<li class="menu"><a class="menu" href="../rss07/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss07/authors.html">Authors</a></li>
+<li class="label">RSS VI</span></li>
+<li class="menu"><a class="menu" href="../rss06/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss06/authors.html">Authors</a></li>
+<li class="label">RSS V</span></li>
+<li class="menu"><a class="menu" href="../rss05/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss05/authors.html">Authors</a></li>
+<li class="label">RSS IV</span></li>
+<li class="menu"><a class="menu" href="../rss04/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss04/authors.html">Authors</a></li>
+<li class="label">RSS III</span></li>
+<li class="menu"><a class="menu" href="../rss03/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss03/authors.html">Authors</a></li>
+<li class="label">RSS II</span></li>
+<li class="menu"><a class="menu" href="../rss02/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss02/authors.html">Authors</a></li>
+<li class="label">RSS I</span></li>
+<li class="menu"><a class="menu" href="../rss01/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss01/authors.html">Authors</a></li>
+<li class="label">Ethics and Malpractice</span></li>
+<li class="menu"><a class="menu" href="../statement.html">Statement</a></li>
+<li class="label">Contact</span></li>
+<li class="last"><a class="menu" href="../contact.html">Webmaster</a></li>
+<li class="no-style"><center><img border="0" src="../robot-small.jpg"></center>
+<br><br><br>
+<center>We are using google analytics for statistics!</center>
+</ul>
+</div>  
+<div class="content">  
+  
+<h1>Robotics: Science and Systems XXI</h1>  
+<h3>Learned Perceptive Forward Dynamics Model for Safe and Platform-aware Robotic Navigation</h3> 
+<i>Pascal Roth, Jonas Frey, Cesar Cadena, Marco Hutter</i><br> 
+<p>  
+<b>Abstract:</b>  
+</p><p style="text-align: justify;">  
+Ensuring safe navigation in complex environments requires accurate real-time traversability assessment and understanding of environmental interactions relative to the robot's capabilities. Traditional methods, which assume simplified dynamics, often require designing and tuning cost functions to safely guide paths or actions toward the goal. This process is tedious, environment-dependent, and not generalizable. To overcome these issues, we propose a novel learned perceptive Forward Dynamics Model (FDM) that predicts the robot's future state conditioned on the surrounding geometry and history of proprioceptive measurements, proposing a more scalable, safer, and heuristic-free solution. The FDM is trained on multiple years of simulated navigation experience, including high-risk maneuvers, and real-world interactions to incorporate the full system dynamics beyond rigid body simulation. We integrate our perceptive FDM into a zero-shot Model Predictive Path Integral (MPPI) planning framework, leveraging the learned mapping between actions, future states, and failure probability. This allows for optimizing a simplified cost function, eliminating the need for extensive cost-tuning to ensure safety. On the legged robot ANYmal, the proposed perceptive FDM improves the position estimation by on average 41% over competitive baselines, which translates into a 27% higher navigation success rate in rough simulation environments.  Moreover, we demonstrate effective sim-to-real transfer and showcase the benefit of training on synthetic and real data. Code and models are made publicly available under https://github.com/leggedrobotics/fdm.
+</p>  
+<p>  
+<b>Download:</b> <a href="p001.pdf" target="_blank" onclick="_gaq.push(['_trackEvent','rss10/p001.pdf','PDF',this.href]);"><img src="../icon-pdf.png" border=0></a> 
+</p>  
+<p>  
+<b>Bibtex:</b>  
+<pre>  
+@INPROCEEDINGS{RothP-RSS-25, 
+    AUTHOR    = {Pascal Roth AND Jonas Frey AND Cesar Cadena AND Marco Hutter}, 
+    TITLE     = {{Learned Perceptive Forward Dynamics Model for Safe and Platform-aware Robotic Navigation}}, 
+    BOOKTITLE = {Proceedings of Robotics: Science and Systems}, 
+    YEAR      = {2025}, 
+    ADDRESS   = {LosAngeles, CA, USA}, 
+    MONTH     = {June}, 
+    DOI       = {10.15607/RSS.2025.XXI.001} 
+} 
+  
+</pre>  
+</p>  
+  
+</div>  
+<div class="analyt"> 
+<script type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-24631810-1']);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</script>
+</div> 
+</body>  
+</html> 

tests/fixtures/rss2025/p002.html

@@ -0,0 +1,155 @@
+<html> 
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />  
+<head>  
+<meta name="citation_title" content="Demonstrating ViSafe: Vision-enabled Safety for High-speed Detect and Avoid" />  
+<meta name="citation_author" content="Parv Kapoor" />  
+<meta name="citation_author" content="Ian Higgins" />  
+<meta name="citation_author" content="Nikhil Varma Keetha" />  
+<meta name="citation_author" content="Jay Patrikar" />  
+<meta name="citation_author" content="Brady Moon" />  
+<meta name="citation_author" content="Zelin Ye" />  
+<meta name="citation_author" content="Yao He" />  
+<meta name="citation_author" content="Ivan Cisneros" />  
+<meta name="citation_author" content="Changliu Liu" />  
+<meta name="citation_author" content="Eunsuk Kang" />  
+<meta name="citation_author" content="Sebastian Scherer" />  
+<meta name="citation_publication_date" content="2025/06/21" />  
+<meta name="citation_conference_title" content="Robotics: Science and Systems XXI" />  
+<meta name="citation_isbn" content="979-8-9902848-1-4" />  
+<meta name="citation_volume" content="21" />  
+<meta name="citation_pdf_url" content="http://www.roboticsproceedings.org/rss21/p002.pdf" />  
+<title>Robotics: Science and Systems XXI - Online Proceedings</title>  
+<link type="text/css" rel="stylesheet" href="../style.css">  
+<!--[if IE 5]> 
+<link type="text/css" rel="stylesheet" href="../ie5.css"> 
+<![endif]-->  
+<!--[if IE 6]> 
+<link type="text/css" rel="stylesheet" href="../ie6.css"> 
+<![endif]-->  
+</head>  
+<body>  
+<div class="menu">  
+<ul>
+<li class="here"><a class="menu" href="../index.html">Home</a></li>
+<li class="label">RSS XXII</span></li>
+<li class="menu"><a class="menu" href="../rss22/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss22/authors.html">Authors</a></li>
+<li class="label">RSS XXI</span></li>
+<li class="menu"><a class="menu" href="../rss21/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss21/authors.html">Authors</a></li>
+<li class="label">RSS XX</span></li>
+<li class="menu"><a class="menu" href="../rss20/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss20/authors.html">Authors</a></li>
+<li class="label">RSS XIX</span></li>
+<li class="menu"><a class="menu" href="../rss19/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss19/authors.html">Authors</a></li>
+<li class="label">RSS XVIII</span></li>
+<li class="menu"><a class="menu" href="../rss18/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss18/authors.html">Authors</a></li>
+<li class="label">RSS XVII</span></li>
+<li class="menu"><a class="menu" href="../rss17/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss17/authors.html">Authors</a></li>
+<li class="label">RSS XVI</span></li>
+<li class="menu"><a class="menu" href="../rss16/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss16/authors.html">Authors</a></li>
+<li class="label">RSS XV</span></li>
+<li class="menu"><a class="menu" href="../rss15/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss15/authors.html">Authors</a></li>
+<li class="label">RSS XIV</span></li>
+<li class="menu"><a class="menu" href="../rss14/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss14/authors.html">Authors</a></li>
+<li class="label">RSS XIII</span></li>
+<li class="menu"><a class="menu" href="../rss13/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss13/authors.html">Authors</a></li>
+<li class="label">RSS XII</span></li>
+<li class="menu"><a class="menu" href="../rss12/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss12/authors.html">Authors</a></li>
+<li class="label">RSS XI</span></li>
+<li class="menu"><a class="menu" href="../rss11/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss11/authors.html">Authors</a></li>
+<li class="label">RSS X</span></li>
+<li class="menu"><a class="menu" href="../rss10/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss10/authors.html">Authors</a></li>
+<li class="label">RSS IX</span></li>
+<li class="menu"><a class="menu" href="../rss09/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss09/authors.html">Authors</a></li>
+<li class="label">RSS VIII</span></li>
+<li class="menu"><a class="menu" href="../rss08/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss08/authors.html">Authors</a></li>
+<li class="label">RSS VII</span></li>
+<li class="menu"><a class="menu" href="../rss07/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss07/authors.html">Authors</a></li>
+<li class="label">RSS VI</span></li>
+<li class="menu"><a class="menu" href="../rss06/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss06/authors.html">Authors</a></li>
+<li class="label">RSS V</span></li>
+<li class="menu"><a class="menu" href="../rss05/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss05/authors.html">Authors</a></li>
+<li class="label">RSS IV</span></li>
+<li class="menu"><a class="menu" href="../rss04/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss04/authors.html">Authors</a></li>
+<li class="label">RSS III</span></li>
+<li class="menu"><a class="menu" href="../rss03/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss03/authors.html">Authors</a></li>
+<li class="label">RSS II</span></li>
+<li class="menu"><a class="menu" href="../rss02/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss02/authors.html">Authors</a></li>
+<li class="label">RSS I</span></li>
+<li class="menu"><a class="menu" href="../rss01/index.html">Content</a></li>
+<li class="menu"><a class="menu" href="../rss01/authors.html">Authors</a></li>
+<li class="label">Ethics and Malpractice</span></li>
+<li class="menu"><a class="menu" href="../statement.html">Statement</a></li>
+<li class="label">Contact</span></li>
+<li class="last"><a class="menu" href="../contact.html">Webmaster</a></li>
+<li class="no-style"><center><img border="0" src="../robot-small.jpg"></center>
+<br><br><br>
+<center>We are using google analytics for statistics!</center>
+</ul>
+</div>  
+<div class="content">  
+  
+<h1>Robotics: Science and Systems XXI</h1>  
+<h3>Demonstrating ViSafe: Vision-enabled Safety for High-speed Detect and Avoid</h3> 
+<i>Parv Kapoor, Ian Higgins, Nikhil Varma Keetha, Jay Patrikar, Brady Moon, Zelin Ye, Yao He, Ivan Cisneros, Changliu Liu, Eunsuk Kang, Sebastian Scherer</i><br> 
+<p>  
+<b>Abstract:</b>  
+</p><p style="text-align: justify;">  
+Assured safe-separation is essential for achieving seamless high-density operation of airborne vehicles in a shared airspace. To equip resource-constrained aerial systems with this safety-critical capability, we present ViSafe, a high-speed vision-only airborne collision avoidance system.  ViSafe offers a full-stack solution to the Detect and Avoid (DAA) problem by tightly integrating a learning-based edge-AI framework with a custom multi-camera hardware prototype designed under SWaP-C constraints. By leveraging perceptual input-focused control barrier functions (CBF) to design, encode, and enforce safety thresholds, ViSafe can provide provably safe runtime guarantees for self-separation in high-speed aerial operations. We evaluate ViSafe’s performance through an extensive test campaign involving both simulated digital twins and real-world flight scenarios.  By independently varying agent types, closure rates, interaction geometries, and environmental conditions (e.g., weather and lighting), we demonstrate that ViSafe consistently ensures self-separation across diverse scenarios.  In first-of-its-kind real-world high-speed collision avoidance tests with closure rates reaching 144 km/h, ViSafe sets a new benchmark for vision-only autonomous collision avoidance, establishing a new standard for safety in high-speed aerial navigation.
+</p>  
+<p>  
+<b>Download:</b> <a href="p002.pdf" target="_blank" onclick="_gaq.push(['_trackEvent','rss10/p002.pdf','PDF',this.href]);"><img src="../icon-pdf.png" border=0></a> 
+</p>  
+<p>  
+<b>Bibtex:</b>  
+<pre>  
+@INPROCEEDINGS{KapoorP-RSS-25, 
+    AUTHOR    = {Parv Kapoor AND Ian Higgins AND Nikhil Varma Keetha AND Jay Patrikar AND Brady Moon AND Zelin Ye AND Yao He AND Ivan Cisneros AND Changliu Liu AND Eunsuk Kang AND Sebastian Scherer}, 
+    TITLE     = {{Demonstrating ViSafe: Vision-enabled Safety for High-speed Detect and Avoid}}, 
+    BOOKTITLE = {Proceedings of Robotics: Science and Systems}, 
+    YEAR      = {2025}, 
+    ADDRESS   = {LosAngeles, CA, USA}, 
+    MONTH     = {June}, 
+    DOI       = {10.15607/RSS.2025.XXI.002} 
+} 
+  
+</pre>  
+</p>  
+  
+</div>  
+<div class="analyt"> 
+<script type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-24631810-1']);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</script>
+</div> 
+</body>  
+</html> 

tests/fixtures/rss2025/receipts.json

@@ -0,0 +1,65 @@
+{
+  "scope": "Six unmodified official HTML fixtures; exact observations from full167 server freeze. No research PDFs.",
+  "receipts": [
+    {
+      "file": "index.html",
+      "url": "https://www.roboticsproceedings.org/rss21/",
+      "final_url": "https://www.roboticsproceedings.org/rss21/",
+      "http_status": 200,
+      "content_type": "text/html",
+      "bytes": 77753,
+      "sha256": "5d8e530b5ce7184795c14a7dca615f9c1a8b0b189661e0e6e795167b68b8516d",
+      "at": "2026-11-09T09:53:09.121733+00:00"
+    },
+    {
+      "file": "accepted.html",
+      "url": "https://roboticsconference.org/2025/program/papers/",
+      "final_url": "https://roboticsconference.org/2025/program/papers/",
+      "http_status": 200,
+      "content_type": "text/html",
+      "bytes": 197708,
+      "sha256": "fa1a5bce496a85eca8cec79aed8dbc148cda791fe6859375ba0c294983a18683",
+      "at": "2026-11-09T09:53:10.171450+00:00"
+    },
+    {
+      "file": "demos.html",
+      "url": "https://roboticsconference.org/2025/program/demos/",
+      "final_url": "https://roboticsconference.org/2025/program/demos/",
+      "http_status": 200,
+      "content_type": "text/html",
+      "bytes": 43542,
+      "sha256": "0b154ed110f6d49ebe2ba87f2c9c498df5fc1ea00bfe508b548522dfe4779698",
+      "at": "2026-11-09T09:53:11.145288+00:00"
+    },
+    {
+      "file": "authorinfo.html",
+      "url": "https://roboticsconference.org/2025/information/authorinfo/",
+      "final_url": "https://roboticsconference.org/2025/information/authorinfo/",
+      "http_status": 200,
+      "content_type": "text/html",
+      "bytes": 54508,
+      "sha256": "24d023daf9a074434c6e004001b863dfd260330e523671d614fa120b052927ab",
+      "at": "2026-11-09T09:53:12.170246+00:00"
+    },
+    {
+      "file": "p001.html",
+      "url": "https://www.roboticsproceedings.org/rss21/p001.html",
+      "final_url": "https://www.roboticsproceedings.org/rss21/p001.html",
+      "http_status": 200,
+      "content_type": "text/html",
+      "bytes": 9048,
+      "sha256": "973172348096a5cbf3e311bdc1ce33b27447761fa1058a050e611aa17547d821",
+      "at": "2026-11-09T09:53:13.437943+00:00"
+    },
+    {
+      "file": "p002.html",
+      "url": "https://www.roboticsproceedings.org/rss21/p002.html",
+      "final_url": "https://www.roboticsproceedings.org/rss21/p002.html",
+      "http_status": 200,
+      "content_type": "text/html",
+      "bytes": 9335,
+      "sha256": "30b5b68e7ed6a809ca51282c5efd1416513417ba91eba2845f2e73ed18b90ba4",
+      "at": "2026-11-09T09:53:14.697857+00:00"
+    }
+  ]
+}

tests/test_aamas_proceedings.py

@@ -0,0 +1,151 @@
+"""Publisher evidence, section boundaries and native-ingestion regression checks."""
+import hashlib
+import json
+import re
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors import aamas
+from paper_atlas.ingestion.records import ingest_file
+
+
+def paper(pid, abstract=True):
+    summary = '<span class="abstract">Research with <i>evidence</i>.<br>More detail.</span>' if abstract else ''
+    return f'''<div class="paper" id="{pid}"><span class="id">{pid}</span>
+    <p class="paper"><span class="title"><a href="../papers/{pid}.pdf">{summary}
+    A structured &amp; auditable paper {pid}</a></span><span class="authors">
+    <a href="authors.html#Jane">Jane Doe<span class="amount">has 7 papers</span></a>, Bob Smith
+    </span></p></div>'''
+
+
+def page(section, pid, abstract=True):
+    heading = aamas.SECTIONS[section][0]
+    return (f'<html><head><title>{heading} - AAMAS 2012</title></head><body>'
+            f'<h1>{heading}</h1><h2 class="section" id="s-1A">Session 1A – Robotics I</h2>'
+            +paper(pid, abstract)+'</body></html>')
+
+
+class AamasProceedingsTests(unittest.TestCase):
+    def snapshots(self, root):
+        root.mkdir()
+        receipts = []
+        items = {'introduction': ('<title>AAMAS 2012</title><p>Aggregate withdrawals do not identify papers.</p>', aamas.ROOT)}
+        for index, section in enumerate(aamas.SECTIONS):
+            items[section] = (page(section, f'P{index}_1', section != 'extabstracts'), aamas.ROOT+'resources/'+section+'.html')
+        for key, (html, url) in items.items():
+            data = html.encode()
+            (root/(key+'.html')).write_bytes(data)
+            receipts.append({'url': url, 'status': 200, 'file': key+'.html', 'bytes': len(data),
+                             'sha256': hashlib.sha256(data).hexdigest(), 'finished_at': '2026-11-09T00:00:00+00:00'})
+        (root/'receipts.json').write_text(json.dumps({'receipts': receipts}))
+        return root
+
+    def test_hidden_abstract_and_author_badges_are_not_title_or_authors(self):
+        rows = aamas.parse_section(page('fullpapers', '1A_1'), 'fullpapers', '2026-11-09T00:00:00Z', {'sha256': 'proof'})
+        row, = rows
+        self.assertEqual(row['title'], 'A structured & auditable paper 1A_1')
+        self.assertEqual(row['abstract'], 'Research with evidence. More detail.')
+        self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith'])
+        self.assertEqual(row['metadata']['authors_display'], 'Jane Doe, Bob Smith')
+        self.assertEqual(row['url'], aamas.ROOT+'resources/fullpapers.html#1A_1')
+        self.assertEqual(row['metadata']['paper_url'], aamas.ROOT+'papers/1A_1.pdf')
+        self.assertEqual(row['metadata']['source_evidence'], {'sha256': 'proof'})
+
+    def test_publication_formats_and_session_topics_do_not_invent_submission_tracks(self):
+        for section, (_, fmt, track, _) in aamas.SECTIONS.items():
+            row, = aamas.parse_section(page(section, 'P_1', False), section, '2026-11-09T00:00:00Z', {})
+            self.assertEqual((row['track'], row['metadata']['publication_format']), (track, fmt))
+            self.assertIsNone(row['abstract'])
+            self.assertEqual(row['status'], 'accepted')
+            self.assertEqual(row['metadata']['decision_date_precision'], 'not_supplied')
+            self.assertEqual(row['metadata']['session']['title'], 'Session 1A – Robotics I')
+            self.assertIn({'namespace': 'aamas_proceedings:publication_format', 'value': fmt}, row['tags'])
+        self.assertEqual(aamas.SECTIONS['fullpapers'][2], 'proceedings_unspecified')
+        self.assertEqual(aamas.SECTIONS['extabstracts'][2], 'proceedings_unspecified')
+
+    def test_changed_edition_section_duplicate_id_or_pdf_link_is_rejected(self):
+        html = page('fullpapers', '1A_1')
+        for changed in [html.replace('AAMAS 2012', 'AAMAS 2025'),
+                        html.replace('<h1>Full Papers</h1>', '<h1>Rejected Papers</h1>'),
+                        html.replace('</body>', paper('1A_1')+'</body>'),
+                        html.replace('../papers/1A_1.pdf', 'https://example.org/1A_1.pdf'),
+                        html.replace('../papers/1A_1.pdf', '../papers/1A_2.pdf')]:
+            with self.subTest(changed=changed):
+                with self.assertRaises(ValueError):
+                    aamas.parse_section(changed, 'fullpapers', '2026-11-09T00:00:00Z', {})
+
+    def test_frozen_responses_require_checksums_success_and_safe_paths(self):
+        for failure in ['checksum', 'status', 'path']:
+            with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary:
+                root = self.snapshots(Path(temporary)/'source')
+                path = root/'receipts.json'
+                receipts = json.loads(path.read_text())
+                item = receipts['receipts'][0]
+                if failure == 'checksum': item['sha256'] = '0'*64
+                elif failure == 'status': item['status'] = 404
+                else: item['file'] = '../outside.html'
+                path.write_text(json.dumps(receipts))
+                with self.assertRaises(ValueError):
+                    aamas.read_pages(snapshots=root)
+
+    def test_reviewed_section_counts_and_year_are_required(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            source = self.snapshots(root/'source')
+            with self.assertRaises(ValueError):
+                aamas.collect(2012, root/'out', snapshots=source)
+            self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
+            with self.assertRaises(ValueError):
+                aamas.collect(2025, root/'future', snapshots=source)
+            self.assertFalse((root/'future').exists())
+
+    def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            source = self.snapshots(root/'source')
+            receipts_path = source/'receipts.json'
+            receipts = json.loads(receipts_path.read_text())
+            for item in receipts['receipts'][1:]:
+                path = source/item['file']
+                data = re.sub(r'P[0-9]_1', 'SAME_1', path.read_text()).encode()
+                path.write_bytes(data)
+                item.update(bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
+            receipts_path.write_text(json.dumps(receipts))
+            small = {key: (*value[:3], 1) for key, value in aamas.SECTIONS.items()}
+            with patch.dict(aamas.SECTIONS, small, clear=True), self.assertRaises(ValueError):
+                aamas.collect(2012, root/'out', snapshots=source)
+            self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
+
+    def test_collection_and_native_ingestion_preserve_formats_evidence_and_missing_abstracts(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            source = self.snapshots(root/'source')
+            small = {key: (*value[:3], 1) for key, value in aamas.SECTIONS.items()}
+            with patch.dict(aamas.SECTIONS, small, clear=True):
+                result = aamas.collect(2012, root/'out', snapshots=source)
+            self.assertEqual(result['records'], 4)
+            self.assertEqual(len(result['files']), 3)
+            self.assertFalse(result['database_written'])
+            self.assertFalse(result['pdfs_downloaded'])
+            db = root/'sample.sqlite'
+            for manifest in result['files']:
+                path = Path(manifest['file'])
+                self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), manifest['file_sha256'])
+                for raw in path.read_text().splitlines():
+                    record = json.loads(raw)
+                    evidence = record['metadata']['source_evidence']
+                    self.assertEqual(hashlib.sha256((root/'out'/evidence['file']).read_bytes()).hexdigest(), evidence['sha256'])
+                ingest_file(db, path)
+            with sqlite3.connect(db) as connection:
+                self.assertEqual(connection.execute('SELECT count(*) FROM papers').fetchone()[0], 4)
+                self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0], 4)
+                self.assertEqual(connection.execute("SELECT count(*) FROM decisions WHERE status='accepted'").fetchone()[0], 4)
+                self.assertEqual(connection.execute('SELECT count(*) FROM papers WHERE abstract IS NULL').fetchone()[0], 1)
+                metadata = [json.loads(r[0]) for r in connection.execute('SELECT metadata_json FROM source_records')]
+                self.assertEqual({r['publication_format'] for r in metadata}, {'full_paper','extended_abstract','demo','doctoral_abstract'})
+                namespaces = {r[0] for r in connection.execute('SELECT DISTINCT namespace FROM tags')}
+                self.assertEqual(namespaces, {'aamas_proceedings:publication_format', 'aamas_proceedings:session'})
+                self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(), [])

tests/test_aamas_publication_identity.py

@@ -0,0 +1,89 @@
+"""Cross-source publisher identities must preserve attempts and decision history."""
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+
+from paper_atlas.db.submissions import identities_for_row
+from paper_atlas.ingestion.records import ingest_file
+
+
+DOI = '10.1234/aamas.identity.example'
+
+
+def row(source, external_id, *, venue='AAMAS', year=2012, track='demo', status='accepted', identifiers=None):
+    return {'source': source, 'external_id': external_id,
+            'url': 'https://example.org/'+source+'/'+external_id,
+            'title': 'A shared publication with independently recorded evidence',
+            'abstract': 'An independently recorded source abstract.',
+            'authors': ['Jane Doe', 'Bob Smith'],
+            'identifiers': identifiers if identifiers is not None else {'doi': DOI},
+            'venue': venue, 'year': year, 'track': track, 'status': status,
+            'observed_at': '2026-11-09T00:00:00Z',
+            'raw_decision': source+' observed '+status,
+            'metadata': {'original_source': source}, 'tags': []}
+
+
+def ingest(db, root, record):
+    path = root/(record['source']+'-'+record['external_id'].replace('/', '-')+'.jsonl')
+    path.write_text(json.dumps(record)+'\n')
+    path.with_suffix('.manifest.json').write_text(json.dumps({
+        'source': record['source'], 'parser_version': 1, 'venue': record['venue'],
+        'year': record['year'], 'track': record['track'], 'count': 1,
+        'list_url': 'https://example.org/'+record['source'], 'status': 'partial',
+        'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
+        'missing_abstract_count': 0}))
+    ingest_file(db, path)
+
+
+class AamasPublicationIdentityTests(unittest.TestCase):
+    def test_only_accepted_known_publisher_rows_supply_publication_doi_identity(self):
+        record = row('aamas_proceedings', '2012/D1_1', identifiers={'doi': 'https://doi.org/'+DOI.upper()})
+        self.assertEqual(identities_for_row(record), [('publication_doi', DOI)])
+        for status in ['rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown']:
+            with self.subTest(status=status):
+                self.assertEqual(identities_for_row({**record, 'status': status}), [])
+        self.assertEqual(identities_for_row({**record, 'source': 'unverified_source'}), [])
+
+    def test_same_edition_doi_joins_sources_in_both_orders_without_duplicate_attempts(self):
+        publisher = row('aamas_proceedings', '2012/D1_1')
+        bibliography = row('dblp', 'conf/atal/Example12')
+        for order in [(publisher, bibliography), (bibliography, publisher)]:
+            with self.subTest(first=order[0]['source']), tempfile.TemporaryDirectory() as temporary:
+                root = Path(temporary)
+                db = root/'sample.sqlite'
+                for record in order:
+                    ingest(db, root, record)
+                ingest(db, root, publisher)
+                with sqlite3.connect(db) as connection:
+                    self.assertEqual(connection.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+                    self.assertEqual(connection.execute('SELECT count(*) FROM source_records').fetchone()[0], 2)
+                    self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0], 1)
+                    self.assertEqual(connection.execute('SELECT count(*) FROM submission_sources').fetchone()[0], 2)
+                    self.assertEqual(connection.execute('SELECT count(*) FROM decision_sources').fetchone()[0], 2)
+                    self.assertEqual(connection.execute('SELECT scheme,value FROM submission_identifiers').fetchall(), [('publication_doi', DOI)])
+                    metadata = [json.loads(r[0]) for r in connection.execute('SELECT metadata_json FROM source_records')]
+                    self.assertEqual({r['original_source'] for r in metadata}, {'aamas_proceedings', 'dblp'})
+                    self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(), [])
+
+    def test_cross_venue_rejection_and_distinct_tracks_remain_separate_attempts(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            db = root/'sample.sqlite'
+            rejected = row('openreview_snapshot', 'original-forum', venue='AAAI', year=2011,
+                           track='main', status='rejected', identifiers={'arxiv': '1101.00001', 'openreview': 'original-forum'})
+            accepted = row('aamas_proceedings', '2012/D1_1', identifiers={'arxiv': '1101.00001', 'doi': DOI})
+            another_track = row('aamas_proceedings', '2012/F1_1', track='proceedings_unspecified', identifiers={'doi': DOI})
+            for record in [rejected, accepted, another_track]:
+                ingest(db, root, record)
+            with sqlite3.connect(db) as connection:
+                self.assertEqual(connection.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+                self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0], 3)
+                history = connection.execute('''SELECT v.acronym,e.year,e.track,d.status FROM decisions d
+                    JOIN submissions s ON s.id=d.submission_id JOIN editions e ON e.id=s.edition_id
+                    JOIN venues v ON v.id=e.venue_id ORDER BY v.acronym,e.track''').fetchall()
+                self.assertEqual(set(history), {('AAAI',2011,'main','rejected'),('AAMAS',2012,'demo','accepted'),('AAMAS',2012,'proceedings_unspecified','accepted')})
+                self.assertEqual(connection.execute('SELECT count(*) FROM decision_sources').fetchone()[0], 3)
+                self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(), [])

tests/test_annotations.py

@@ -0,0 +1,227 @@
+from contextlib import closing, redirect_stdout
+import io
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+import uuid
+
+from paper_atlas import querying
+from paper_atlas.curation.annotations import (assert_annotation, available, initialize, main,
+                                             observations, retract_annotation)
+from paper_atlas.db.backend import open_store
+from paper_atlas.db.store import Store
+from paper_atlas.postgres import schema
+from paper_atlas.postgres.client import connect
+from paper_atlas.postgres.migration import migrate
+
+
+class AnnotationTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
+        self.db = Path(self.tmp.name) / 'sample.sqlite'
+        self.store = Store(self.db); self.addCleanup(self.store.conn.close)
+        self.c = self.store.conn
+        self.paper, self.record = self.store.record('fixture', 'one', 'https://example.org/one', 'First research', None, [])
+        self.other, self.other_record = self.store.record('fixture', 'two', 'https://example.org/two', 'Other research', None, [])
+        self.store.record_tags(self.paper, self.record, 'fixture', [{'namespace':'fixture:topic','value':'original'}])
+        self.c.commit()
+
+    def add(self, **changes):
+        args = dict(paper=self.paper, kind='tag', namespace='custom:topic', value='reading',
+                    request_id='request-one', evidence_key='manual:reading', reason='Reviewed metadata', actor='reviewer')
+        args.update(changes)
+        return assert_annotation(self.c, **args)
+
+    def query(self, *args):
+        self.c.commit()
+        with redirect_stdout(io.StringIO()) as output:
+            querying.main(['--db', str(self.db), *map(str, args)])
+        return json.loads(output.getvalue())
+
+    def test_read_does_not_install_extension_and_write_requires_explicit_init(self):
+        self.assertFalse(available(self.c))
+        self.assertEqual(observations(self.c, self.paper), [])
+        with self.assertRaisesRegex(RuntimeError, 'init explicitly'):
+            self.add()
+        self.assertEqual(self.query('--research-type', 'custom:method=empirical'), [])
+        self.assertFalse(available(self.c))
+        self.assertEqual(len(schema.inspect(self.c)), 21)
+
+    def test_schema_upgrade_is_transactional_and_marker_mismatch_fails_closed(self):
+        self.assertTrue(initialize(self.c)); self.c.rollback()
+        self.assertFalse(available(self.c))
+        self.c.execute("INSERT INTO schema_migrations(name) VALUES('annotation_events_v1')")
+        with self.assertRaisesRegex(RuntimeError, 'mismatch'):
+            initialize(self.c)
+        self.c.rollback()
+        self.assertTrue(initialize(self.c)); self.assertFalse(initialize(self.c))
+        self.assertEqual(len(schema.inspect(self.c)), 22)
+
+    def test_unmarked_table_cannot_receive_assertions(self):
+        initialize(self.c)
+        self.c.execute("DELETE FROM schema_migrations WHERE name='annotation_events_v1'")
+        with self.assertRaisesRegex(RuntimeError, 'init explicitly'):
+            self.add()
+        self.assertEqual(self.c.execute('SELECT count(*) FROM annotation_events').fetchone()[0], 0)
+
+    def test_types_tags_and_platform_labels_are_independent_and_queryable(self):
+        initialize(self.c)
+        self.add()
+        self.add(kind='research_type', namespace='custom:topic', request_id='type', value='reading')
+        self.assertEqual(len(self.query('--tag', 'custom:topic=reading')), 1)
+        self.assertEqual(len(self.query('--research-type', 'custom:topic=reading')), 1)
+        self.assertEqual(len(self.query('--tag', 'fixture:topic=original')), 1)
+        detail = self.query('--paper', self.paper)[0]
+        self.assertEqual(len(detail['annotation_tags']), 1)
+        self.assertEqual(len(detail['research_types']), 1)
+        self.assertNotIn('annotation_history', detail)
+        self.assertEqual(detail['tags'][0]['namespace'], 'fixture:topic')
+
+    def test_repeated_request_is_idempotent_and_changed_content_fails(self):
+        initialize(self.c); event = self.add()
+        self.assertEqual(self.add(), event)
+        for change in ({'value':'different'}, {'kind':'research_type'}, {'reason':'changed'}, {'paper':self.other}):
+            with self.assertRaisesRegex(ValueError, 'different annotation'):
+                self.add(**change)
+        self.assertEqual(self.c.execute('SELECT count(*) FROM annotation_events').fetchone()[0], 1)
+
+    def test_retraction_keeps_original_and_other_assertions_of_same_label(self):
+        initialize(self.c); first = self.add(); second = self.add(request_id='second', evidence_key='independent')
+        kwargs = dict(request_id='retract', reason='No longer relevant', actor='reviewer')
+        removed = retract_annotation(self.c, first, **kwargs)
+        self.assertEqual(retract_annotation(self.c, first, **kwargs), removed)
+        self.assertEqual([r['id'] for r in observations(self.c, self.paper)], [second])
+        self.assertEqual(len(self.query('--paper', self.paper, '--annotation-history')[0]['annotation_history']), 3)
+        with self.assertRaisesRegex(ValueError, 'already retracted'):
+            retract_annotation(self.c, first, **{**kwargs,'request_id':'duplicate'})
+        with self.assertRaisesRegex(ValueError, 'existing assertion'):
+            retract_annotation(self.c, removed, **{**kwargs,'request_id':'nested'})
+
+    def test_source_assertions_require_matching_paper_and_platform(self):
+        initialize(self.c)
+        for change in ({'namespace':'fixture:type'}, {'namespace':'other:type','source_record_id':self.record},
+                       {'namespace':'fixture:type','source_record_id':self.other_record}):
+            with self.assertRaises(ValueError):
+                self.add(**change)
+        event = self.add(kind='research_type', namespace='fixture:declared_type', source_record_id=self.record)
+        with self.assertRaisesRegex(ValueError, 'Manual CLI'):
+            retract_annotation(self.c, event, request_id='cannot', reason='x', actor='y', custom_only=True)
+        self.assertEqual(observations(self.c, self.paper)[0]['source_record_id'], self.record)
+
+    def test_platform_tag_refresh_preserves_both_custom_kinds(self):
+        initialize(self.c); self.add(); self.add(kind='research_type', request_id='type')
+        self.store.record_tags(self.paper, self.record, 'fixture', [{'namespace':'fixture:topic','value':'refreshed'}])
+        self.assertEqual(len(observations(self.c, self.paper)), 2)
+        self.assertEqual(len(self.query('--tag','fixture:topic=refreshed')), 1)
+        self.assertEqual(self.query('--tag','fixture:topic=original'), [])
+
+    def test_merge_preserves_retractions_evidence_and_original_paper_ids(self):
+        initialize(self.c); event = self.add(source_record_id=self.record)
+        removed = retract_annotation(self.c, event, request_id='withdraw', reason='corrected', actor='reviewer')
+        active = self.add(request_id='active', kind='research_type', value='survey')
+        self.store.merge(self.other, self.paper, 'Independently reviewed identity')
+        rows = observations(self.c, self.other, history=True)
+        self.assertEqual([r['id'] for r in rows], [event, removed, active])
+        self.assertEqual({r['original_paper_id'] for r in rows}, {self.paper})
+        self.assertEqual({r['paper_id'] for r in rows}, {self.other})
+        self.assertEqual([r['id'] for r in observations(self.c, self.other)], [active])
+        self.assertEqual(self.add(source_record_id=self.record), event)  # exact retry survives canonical merge
+        self.assertEqual(self.c.execute('PRAGMA foreign_key_check').fetchall(), [])
+
+    def test_invalid_input_does_not_create_events(self):
+        initialize(self.c)
+        for change in ({'paper':True}, {'paper':999999}, {'kind':'main'}, {'namespace':'arxiv'},
+                       {'namespace':'custom:topic;DROP'}, {'value':' '}, {'reason':''},
+                       {'actor':''}, {'request_id':''}, {'evidence_key':''}, {'source_record_id':True},
+                       {'evidence_url':'file:///etc/passwd'}, {'evidence_url':'https://user:secret@example.org/'}):
+            with self.subTest(change=change), self.assertRaises(ValueError):
+                self.add(**change)
+        self.assertEqual(self.c.execute('SELECT count(*) FROM annotation_events').fetchone()[0], 0)
+
+    def test_cli_manual_scope_and_complete_assert_retract_lifecycle(self):
+        with redirect_stdout(io.StringIO()):
+            main(['init','--db',str(self.db)])
+            main(['assert','--db',str(self.db),'--paper',str(self.paper),'--kind','research_type',
+                  '--namespace','custom:method','--value','empirical','--evidence-key','manual:review',
+                  '--request-id','cli-one','--reason','Read abstract','--actor','reviewer'])
+        self.assertEqual(len(self.query('--research-type','custom:method=empirical')), 1)
+        event = observations(self.c, self.paper)[0]['id']
+        with redirect_stdout(io.StringIO()):
+            main(['retract','--db',str(self.db),'--event',str(event),'--request-id','cli-two',
+                  '--reason','Corrected classification','--actor','reviewer'])
+        self.assertEqual(self.query('--research-type','custom:method=empirical'), [])
+        with redirect_stdout(io.StringIO()) as out:
+            main(['list','--db',str(self.db),'--paper',str(self.paper),'--history'])
+        self.assertEqual(len(json.loads(out.getvalue())), 2)
+
+
+@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
+class PostgreSQLAnnotationTests(unittest.TestCase):
+    def setUp(self):
+        self.service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
+        self.target = 'pa_anno_' + uuid.uuid4().hex[:12]
+        self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
+        self.db = Path(self.tmp.name) / 'source.sqlite'
+        self.s = Store(self.db)
+        self.paper, self.record = self.s.record('fixture','one','https://example.org','Paper',None,[])
+        self.s.conn.commit()
+        self.addCleanup(self.s.conn.close); self.addCleanup(self.cleanup)
+
+    def cleanup(self):
+        with connect(self.service) as c:
+            for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (self.target,)):
+                c.execute('DROP SCHEMA ' + schema.quote(name) + ' CASCADE')
+
+    def migrate(self):
+        self.s.conn.commit()
+        self.s.conn.close()  # Freeze/checkpoint the fixture, just like a real source migration.
+        with redirect_stdout(io.StringIO()):
+            return migrate(self.db, self.service, Path(self.tmp.name)/'report.json', schema=self.target, min_free_gib=0)
+
+    def args(self, **changes):
+        a = dict(paper=self.paper,kind='research_type',namespace='custom:method',value='survey\x00and\x01',
+                 request_id='native',evidence_key='manual',reason='Reviewed\x00evidence',actor='reviewer')
+        a.update(changes); return a
+
+    def test_native_schema_upgrade_roundtrip_retraction_and_queries(self):
+        self.migrate()
+        with closing(open_store(pg_service=self.service, pg_schema=self.target).conn) as c:
+            self.assertTrue(initialize(c)); self.assertFalse(initialize(c))
+            event = assert_annotation(c, **self.args()); c.commit()
+            self.assertEqual(assert_annotation(c, **self.args()), event)
+            self.assertEqual(observations(c,self.paper)[0]['value'], 'survey\x00and\x01')
+            c.commit()
+        with redirect_stdout(io.StringIO()) as out:
+            querying.main(['--pg-service',self.service,'--pg-schema',self.target,'--paper',str(self.paper)])
+        self.assertEqual(json.loads(out.getvalue())[0]['research_types'][0]['id'], event)
+        with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c:
+            retract_annotation(c,event,request_id='removed',reason='Changed',actor='reviewer');c.commit()
+            self.assertEqual(observations(c,self.paper), [])
+            self.assertEqual(len(observations(c,self.paper,history=True)), 2)
+
+    def test_initialized_sqlite_extension_migrates_all_rows_hashes_and_identity(self):
+        initialize(self.s.conn)
+        event = assert_annotation(self.s.conn, **self.args())
+        retract_annotation(self.s.conn,event,request_id='retracted',reason='Corrected',actor='reviewer')
+        report = self.migrate()
+        self.assertEqual(len(report['tables']),22)
+        self.assertEqual(report['tables']['annotation_events']['source_rows'],2)
+        self.assertEqual(report['tables']['annotation_events']['source_sha256'],report['tables']['annotation_events']['target_sha256'])
+        with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c:
+            new = assert_annotation(c, **self.args(request_id='after-migration'))
+            self.assertGreater(new,2); c.commit()
+            self.assertEqual(len(observations(c,self.paper,history=True)),3)
+
+    def test_native_upgrade_rollback_keeps_legacy_schema_usable(self):
+        self.migrate()
+        with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c:
+            initialize(c); c.rollback()
+            self.assertFalse(available(c))
+            self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
+            self.assertTrue(initialize(c))
+            with self.assertRaises(ValueError):
+                assert_annotation(c, **self.args(namespace='wrong:type',source_record_id=self.record))
+            self.assertEqual(c.execute('SELECT count(*) FROM annotation_events').fetchone()[0],0)
+            c.rollback()

tests/test_coverage_report.py

@@ -7,6 +7,34 @@ from paper_atlas.audits.coverage import report
 from paper_atlas.db.store import Store
 
 
+def add_coverage_evidence(store):
+    paper, record = store.record('official_metadata', 'known', 'https://example.org/known',
+        'Known paper with two source observations', 'Complete canonical abstract', ['Jane Doe', 'John Smith'],
+        {'openreview': 'coverageForum', 'arxiv': '2401.98765'})
+    edition = store.decision(paper, record, 'COLING+LREC', 2024, 'main', 'known',
+        'https://example.org/accept', 'accepted', 'Accept', aliases=['COLING', 'LREC'],
+        identities=[('openreview', 'coverageForum')], observed_at='2025-05-01T00:00:00Z')
+    same, mirror = store.record('bibliography', 'mirror', 'https://example.org/mirror',
+        'Known paper with two source observations', None, ['Jane Doe', 'John Smith'], {'openreview': 'coverageForum'})
+    store.decision(same, mirror, 'COLING+LREC', 2024, 'main', 'mirror', 'https://example.org/older',
+        'rejected', 'Reject', identities=[('openreview', 'coverageForum')],
+        observed_at='2025-05-01T00:00:00Z', decision_date='2025-04-30T00:00:00Z')
+    unknown, unknown_record = store.record('unknown_catalogue', 'unknown', 'https://example.org/unknown',
+        'Indexed work without a public decision', None, [])
+    store.decision(unknown, unknown_record, 'COLING+LREC', 2024, 'main', 'unknown',
+        'https://example.org/unknown', None, '', identities=[('openreview', 'unknownForum')])
+    same, later = store.record('official_metadata', 'later', 'https://example.org/later',
+        'Known paper with two source observations', 'Complete canonical abstract', ['Jane Doe', 'John Smith'],
+        {'openreview': 'coverageNext', 'arxiv': '2401.98765'})
+    store.decision(same, later, 'COLING+LREC', 2025, 'findings', 'later', 'https://example.org/later',
+        'accepted', 'Accept', identities=[('openreview', 'coverageNext')], observed_at='2025-06-01T00:00:00Z')
+    source = store.source('official_metadata', 'https://example.org')
+    for url, status in [('https://example.org/list', 'complete'), ('https://example.org/identity-refresh', 'partial')]:
+        store.conn.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count)
+            VALUES(?,?,?,?,?,?)''', (edition, source, url, status, 1, 1))
+    store.conn.commit()
+
+
 class CoverageTests(unittest.TestCase):
     def setUp(self):
         self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
@@ -31,5 +59,46 @@ class CoverageTests(unittest.TestCase):
         self.store.conn.commit();coling=report(self.db,self.registry)['venues'][0]
         self.assertEqual(coling['imported_submissions'],2);self.assertEqual(coling['distinct_papers'],1)
 
+    def test_observations_attempts_and_missing_source_abstracts_stay_separate(self):
+        add_coverage_evidence(self.store)
+        result = report(self.db, self.registry)
+        self.assertEqual(result['engine'], 'sqlite')
+        self.assertEqual(result['database'], str(self.db))
+        coling = result['venues'][0]
+        self.assertEqual((coling['imported_submissions'], coling['distinct_papers']), (3, 2))
+        edition, findings = coling['editions']
+        self.assertEqual((edition['submissions'], edition['papers'], edition['papers_without_abstract_text']), (2, 2, 1))
+        self.assertEqual(edition['latest_observed_status_counts']['accepted'], 1)
+        self.assertEqual(edition['latest_observed_status_counts']['rejected'], 0)
+        self.assertEqual(edition['latest_observed_status_counts']['unknown'], 1)
+        self.assertEqual(edition['decision_observation_counts']['accepted'], 1)
+        self.assertEqual(edition['decision_observation_counts']['rejected'], 1)
+        self.assertEqual(edition['decision_observation_counts']['unknown'], 0)
+        counts = {row['source']: row for row in edition['source_record_counts']}
+        self.assertEqual(counts['bibliography']['source_records_without_abstract_text'], 1)
+        self.assertEqual(counts['official_metadata']['source_records'], 1)
+        self.assertEqual(len(edition['sources']), 2)
+        self.assertEqual(findings['track'], 'findings')
+        self.assertEqual(result['imported_venues_outside_registry'], [])
+
+    def test_missing_sqlite_audit_does_not_create_a_database(self):
+        path = self.root / 'absent.sqlite'
+        with self.assertRaisesRegex(ValueError, 'Existing database'):
+            report(path, self.registry)
+        self.assertFalse(path.exists())
+
+    def test_empty_editions_remain_visible_after_shared_aggregation(self):
+        self.store.conn.execute("INSERT INTO venues(acronym,name,field,venue_type) VALUES('AAAI','AAAI','general_ai','conference')")
+        venue = self.store.conn.execute("SELECT id FROM venues WHERE acronym='AAAI'").fetchone()[0]
+        self.store.conn.execute("INSERT INTO editions(venue_id,year,track) VALUES(?,2026,'main')", (venue,))
+        self.store.conn.commit()
+        aaai = report(self.db, self.registry)['venues'][1]
+        self.assertEqual(aaai['history_status'], 'no_imported_records')
+        self.assertEqual(len(aaai['editions']), 1)
+        edition = aaai['editions'][0]
+        self.assertEqual((edition['submissions'], edition['papers'], edition['papers_without_abstract_text']), (0, 0, 0))
+        self.assertEqual(sum(edition['latest_observed_status_counts'].values()), 0)
+        self.assertEqual(edition['source_record_counts'], [])
+
 
 if __name__=='__main__':unittest.main()

tests/test_dblp_records.py

@@ -10,7 +10,7 @@ from unittest.mock import patch
 
 from paper_atlas.collectors.dblp import XML_SHA
 from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row,material_reason
-from paper_atlas.collectors.dblp_scope import scope
+from paper_atlas.collectors.dblp_scope import scope, HISTORICAL_WORKSHOP_TITLES
 from paper_atlas.db.store import Store
 from paper_atlas.ingestion.records import ingest_file
 
@@ -70,6 +70,69 @@ class DblpRecordTests(unittest.TestCase):
         self.assertEqual(scope(self.row(),self.volume('kdd',2024))[1],'crossref_venue_conflict')
         self.assertEqual(scope(self.row(year=2027),self.volume('aaai',2027))[1],'future_edition')
 
+    def test_historical_primary_series_workshop_names_require_exact_editions(self):
+        self.assertEqual(set(HISTORICAL_WORKSHOP_TITLES),
+                         {('aistats',year) for year in [1995,1997,1999,2001,2003,2005]} |
+                         {('colt',year) for year in [1988,1989,1990,1991]})
+        for (family,year),prefix in HISTORICAL_WORKSHOP_TITLES.items():
+            with self.subTest(family=family,year=year):
+                title=prefix+f', {family.upper()} {year}, Proceedings.'
+                volume=self.volume(family,year,title)
+                row=self.row(family,family.upper(),year)
+                if year==1995:
+                    row['fields']['year']=['1996'];volume['fields']['year']=['1996']
+                edition,reason=scope(row,volume)
+                self.assertIsNone(reason);self.assertEqual(edition['track'],'main')
+                self.assertEqual(edition['year'],year)
+                self.assertIn('Historical primary series',edition['track_scope_note'])
+                record,reason=normalize_row(row,{volume['key']:volume},{family.upper()})
+                self.assertIsNone(reason);self.assertEqual(record['metadata']['track_scope_evidence_url'],edition['track_scope_evidence_url'])
+                self.assertEqual(record['metadata']['proceedings_title'],title)
+                if year==1995:self.assertEqual(record['metadata']['bibliographic_year'],1996)
+
+    def test_historical_exception_does_not_promote_affiliated_or_wrong_scope_workshops(self):
+        title=HISTORICAL_WORKSHOP_TITLES[('aistats',1995)]+', AISTATS 1995, Proceedings.'
+        for book,key,changed in [
+            ('AISTATS','conf/aistats/1995w',title),
+            ('AISTATS (Workshops)','conf/aistats/1995',title),
+            ('AISTATS','conf/aistats/1995',title.replace('Fifth','Sixth')),
+            ('AISTATS','conf/aistats/1995',title.replace('1995','1997'))]:
+            volume=self.volume('aistats',1995,changed);volume['key']=key
+            row=self.row('aistats',book,1995);row['fields']['crossref']=[key]
+            self.assertEqual(scope(row,volume)[0]['track'],'workshop')
+        volume=self.volume('aistats',2010,'Active Learning and Experimental Design workshop, In conjunction with AISTATS 2010')
+        volume['key']='conf/aistats/2010w'
+        row=self.row('aistats','AISTATS',2011);row['fields']['crossref']=[volume['key']]
+        self.assertEqual(scope(row,volume)[0]['track'],'workshop')
+
+    def test_combined_colt_kernel_volume_keeps_unknown_per_paper_track(self):
+        title='Computational Learning Theory and Kernel Machines, 16th Annual Conference on Computational Learning Theory and 7th Kernel Workshop, COLT/Kernel 2003, Washington, DC, USA, August 24-27, 2003, Proceedings'
+        volume=self.volume('colt',2003,title);row=self.row('colt','COLT',2003)
+        record,reason=normalize_row(row,{volume['key']:volume},{'COLT'})
+        self.assertIsNone(reason);self.assertEqual(record['track'],'proceedings_unspecified')
+        self.assertEqual(record['status'],'accepted')
+        self.assertIn('per-paper section absent',record['metadata']['track_scope_note'])
+        self.assertEqual(record['metadata']['track_scope_evidence_url'],'https://doi.org/10.1007/b12006')
+        volume['key']='conf/colt/2003w';row['fields']['crossref']=[volume['key']]
+        self.assertEqual(scope(row,volume)[0]['track'],'workshop')
+
+    def test_historical_scope_provenance_survives_file_ingestion(self):
+        title=HISTORICAL_WORKSHOP_TITLES[('colt',1988)]+', COLT 1988, Proceedings.'
+        row=self.row('colt','COLT',1988);volume=self.volume('colt',1988,title)
+        record,reason=normalize_row(row,{volume['key']:volume},{'COLT'})
+        self.assertIsNone(reason)
+        with tempfile.TemporaryDirectory() as tmp:
+            root=Path(tmp);path=root/'history.jsonl'
+            path.write_text(json.dumps(record)+'\n')
+            path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','count':1,'list_url':'https://doi.org/10.4230/dblp.xml.2026-11-01','file_sha256':hashlib.sha256(path.read_bytes()).hexdigest()}))
+            ingest_file(root/'sample.sqlite',path)
+            with closing(sqlite3.connect(root/'sample.sqlite')) as connection:
+                track=connection.execute('SELECT track FROM editions').fetchone()[0]
+                metadata=json.loads(connection.execute('SELECT metadata_json FROM source_records').fetchone()[0])
+            self.assertEqual(track,'main')
+            self.assertEqual(metadata['track_scope_evidence_url'],'https://www.learningtheory.org/')
+            self.assertEqual(metadata['proceedings_title'],title)
+
     def test_only_own_identifier_links_are_used_and_ambiguity_is_preserved(self):
         own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/published','https://doi.org/10.48550/arXiv.2401.00001','https://openreview.net/forum?id=realForum'],
                                     'cite':['https://doi.org/10.9999/reference']})

tests/test_edition_tracks.py

@@ -0,0 +1,159 @@
+"""Stable-attempt correction, rollback and stale whole-volume refresh on both engines."""
+from contextlib import closing, redirect_stdout
+import copy
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+import uuid
+
+from paper_atlas.db.backend import open_store
+from paper_atlas.db.edition_scopes import KEY
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.edition_tracks import proposal,apply_plan,verify,validate,run
+from paper_atlas.ingestion.event_identity import database_identity,values
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.postgres.client import connect
+from paper_atlas.postgres.migration import migrate
+from paper_atlas.postgres.schema import quote
+
+
+def seed(db):
+    s=Store(db); rows=[]
+    for i,track in enumerate(['main','survey','doctoral_consortium','proceedings_unspecified'],1):
+        external='entry'+str(i);doi='10.1234/entry'+str(i)
+        p,r=s.record('fixture',external,'https://example.org/'+external,'Publication '+external,
+                     'Abstract with NUL\x00 and SOH\x01',[],{'doi':doi})
+        eid=s.decision(p,r,'JOINT',2022,'main',external,'https://example.org/'+external,'accepted','published',identities=[('doi',doi)],observed_at='2022-01-01T00:00:00Z')
+        sid=s.conn.execute('SELECT id FROM submissions WHERE source_record_id=?',(r,)).fetchone()[0]
+        s.record_tags(p,r,'fixture',[{'namespace':'fixture:subject','value':'Original'}])
+        rows.append(dict(submission_id=sid,source_record_id=r,external_id=external,source='fixture',doi=doi,
+                         from_edition_id=eid,year=2022,venue='JOINT',venue_names=['JOINT'],track=track,
+                         section=track,evidence_url='https://example.org/proceedings/2022/',evidence_sha256='a'*64))
+    source=s.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
+    s.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'complete',4,4,'checkpoint','2022-01-01T00:00:00Z',NULL)",(eid,source,'https://example.org/volume'))
+    s.conn.commit();s.conn.close();return {'format':1,'records':rows}
+
+
+class TrackCases:
+    def store(self):return open_store(**self.target)
+
+    def plan(self,c):return proposal(c,self.inventory,database_identity(c,self.target.get('db')))
+
+    def test_move_preserves_all_history_tags_text_and_stable_identities(self):
+        with closing(self.store().conn)as c:
+            plan=self.plan(c);before=plan['before']['protected'];r=apply_plan(c,plan);c.commit()
+            self.assertEqual((r['events_moved'],r['records_verified']),(3,4))
+            self.assertEqual(r['protected'],before)
+            self.assertEqual(apply_plan(c,plan)['status'],'already_applied')
+            self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count FROM edition_sources ORDER BY edition_id')),[['partial',1,1]]*4)
+            self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
+            self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],4)
+            self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0)
+
+    def test_source_record_refresh_keeps_scope_and_original_attempt(self):
+        with closing(self.store().conn)as c:
+            plan=self.plan(c);apply_plan(c,plan);c.commit()
+        with closing(self.store().conn)as c:
+            # Use Store methods with the same writer connection as production.
+            s=object.__new__(Store);s.conn=c;s.event_identity=True
+            r=self.inventory['records'][1];metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(r['source_record_id'],)).fetchone()[0]);saved=metadata[KEY]
+            p,record=s.record('fixture',r['external_id'],'https://example.org/new','Refreshed title',None,[],{'doi':r['doi']},{'new_source_field':True})
+            eid=s.decision(p,record,'JOINT',2022,'main',r['external_id'],'https://example.org/entry2','accepted','published',identities=[('doi',r['doi'])],observed_at='2022-01-01T00:00:00Z')
+            self.assertEqual(c.execute('SELECT track FROM editions WHERE id=?',(eid,)).fetchone()[0],'survey')
+            self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
+            self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])[KEY],saved)
+            with self.assertRaisesRegex(ValueError,'cannot supply'):
+                s.record('fixture',r['external_id'],'https://example.org/new','Forged',None,[],metadata={KEY:[]})
+            c.rollback()
+
+    def test_stale_snapshot_and_partial_inventory_are_rejected(self):
+        with closing(self.store().conn)as c:
+            plan=self.plan(c)
+            c.execute("UPDATE papers SET title='Changed' WHERE id=(SELECT paper_id FROM submissions WHERE id=?)",(self.inventory['records'][0]['submission_id'],))
+            with self.assertRaisesRegex(ValueError,'Stale'):apply_plan(c,plan)
+            c.rollback()
+            bad=copy.deepcopy(self.inventory);bad['records'].pop()
+            with self.assertRaisesRegex(ValueError,'every attempt'):proposal(c,bad,database_identity(c,self.target.get('db')))
+            bad=copy.deepcopy(self.inventory);bad['records'][0]['doi']='10.1234/wrong'
+            with self.assertRaisesRegex(ValueError,'DOI'):proposal(c,bad,database_identity(c,self.target.get('db')))
+
+    def test_failure_after_updates_rolls_back_all_scope_and_metadata_changes(self):
+        with closing(self.store().conn)as c:
+            plan=self.plan(c)
+            # A late coverage failure must roll back earlier moved attempts.
+            source=c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
+            c.execute("INSERT INTO sources(name,homepage) VALUES('other','https://example.org')")
+            other=c.execute("SELECT id FROM sources WHERE name='other'").fetchone()[0]
+            c.execute('UPDATE edition_sources SET source_id=? WHERE source_id=?',(other,source));c.commit()
+            plan=self.plan(c)
+            try:
+                with self.assertRaisesRegex(ValueError,'outside'):apply_plan(c,plan)
+            finally:c.rollback()
+            self.assertEqual(self.plan(c)['before'],plan['before'])
+            self.assertEqual(c.execute('SELECT count(*) FROM editions').fetchone()[0],1)
+
+    def test_import_old_whole_volume_fails_without_corrupting_coverage(self):
+        with closing(self.store().conn)as c:
+            plan=self.plan(c);apply_plan(c,plan);c.commit()
+        r=self.inventory['records'][1];path=self.root/'old.jsonl'
+        row=dict(source='fixture',external_id=r['external_id'],url='https://example.org/entry2',title='Old title',authors=[],identifiers={'doi':r['doi']},venue='JOINT',year=2022,track='main',status='accepted')
+        path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps(dict(source='fixture',venue='JOINT',year=2022,track='main',count=1,list_url='https://example.org/volume',status='complete')))
+        with self.assertRaisesRegex(ValueError,'Recollect'):ingest_file(self.target.get('db'),path,**{k:v for k,v in self.target.items()if k!='db'})
+        with closing(self.store().conn)as c:verify(c,plan)
+
+    def test_nonempty_target_and_alias_drift_fail_before_moving(self):
+        with closing(self.store().conn)as c:
+            c.execute("INSERT INTO venue_aliases VALUES((SELECT id FROM venues WHERE acronym='JOINT'),'ADDED')")
+            with self.assertRaisesRegex(ValueError,'aliases'):self.plan(c)
+            c.rollback()
+            s=object.__new__(Store);s.conn=c;s.event_identity=True
+            p,r=s.record('fixture','collision','https://example.org/collision','Distinct contribution',None,[],{'doi':'10.1234/collision'})
+            s.decision(p,r,'JOINT',2022,'survey','collision','https://example.org/collision','accepted','published')
+            with self.assertRaisesRegex(ValueError,'Nonempty target'):self.plan(c)
+            c.rollback()
+            self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
+
+    def test_repeated_plan_detects_coverage_or_scope_evidence_drift(self):
+        with closing(self.store().conn)as c:
+            plan=self.plan(c);apply_plan(c,plan);c.commit()
+            c.execute("UPDATE edition_sources SET imported_count=99")
+            with self.assertRaisesRegex(ValueError,'coverage'):apply_plan(c,plan)
+            c.rollback()
+            record=self.inventory['records'][0]['source_record_id']
+            m=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0]);m[KEY]=[]
+            c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(m),record))
+            with self.assertRaisesRegex(ValueError,'unchanged track'):apply_plan(c,plan)
+            c.rollback()
+
+    def test_cli_hash_binding_preflight_and_duplicate_inventory(self):
+        inventory=self.root/'inventory.json';inventory.write_text(json.dumps(self.inventory))
+        planfile=self.root/'plan.json';p=run(self.target.get('db'),planfile,inventory=inventory,**{k:v for k,v in self.target.items()if k!='db'})
+        with self.assertRaisesRegex(ValueError,'hash'):run(self.target.get('db'),self.root/'bad.json',plan=planfile,plan_sha256='0'*64,**{k:v for k,v in self.target.items()if k!='db'})
+        report=run(self.target.get('db'),self.root/'applied.json',plan=planfile,plan_sha256=hashlib.sha256(planfile.read_bytes()).hexdigest(),**{k:v for k,v in self.target.items()if k!='db'})
+        self.assertEqual(report['events_moved'],3)
+        bad=copy.deepcopy(self.inventory);bad['records'].append(bad['records'][0])
+        with self.assertRaisesRegex(ValueError,'Repeated'):validate(bad)
+
+
+class SQLiteTrackTests(TrackCases,unittest.TestCase):
+    def setUp(self):
+        t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite'
+        self.inventory=seed(db);self.target={'db':db}
+
+
+@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL service')
+class PostgreSQLTrackTests(TrackCases,unittest.TestCase):
+    def setUp(self):
+        t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite';self.inventory=seed(db)
+        self.service=os.environ['PAPER_ATLAS_TEST_PG_SERVICE'];self.schema='pa_tracks_'+uuid.uuid4().hex[:12];self.addCleanup(self.cleanup)
+        with redirect_stdout(io.StringIO()):migrate(db,self.service,self.root/'migration.json',schema=self.schema,min_free_gib=0)
+        self.target={'pg_service':self.service,'pg_schema':self.schema}
+
+    def cleanup(self):
+        with connect(self.service)as c:
+            for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(self.schema,)):
+                c.execute('DROP SCHEMA '+quote(name)+' CASCADE')

tests/test_event_identity.py

@@ -0,0 +1,202 @@
+from contextlib import closing, redirect_stdout, contextmanager
+import io
+import json
+import os
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+import uuid
+from importlib.resources import files
+
+from paper_atlas.db.store import Store
+from paper_atlas.db.backend import open_store
+from paper_atlas.db.events import has_event_key
+from paper_atlas.collectors.dblp import XML_SHA
+from paper_atlas.ingestion.event_identity import plan, upgrade, exact_counts
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.ingestion.backfill import backfill
+from paper_atlas.postgres.client import connect
+from paper_atlas.postgres.schema import quote
+from paper_atlas.postgres.migration import migrate
+from paper_atlas import querying
+
+
+@contextmanager
+def opened_store(**kwargs):
+    store=open_store(**kwargs)
+    try:
+        yield store
+    finally:
+        store.conn.close()
+
+
+def legacy(db):
+    sql = files('paper_atlas.db').joinpath('schema.sql').read_text().replace(" event_key TEXT NOT NULL DEFAULT '',\n", '').replace('UNIQUE(venue_id,year,track,event_key)', 'UNIQUE(venue_id,year,track)')
+    c = sqlite3.connect(db); c.executescript(sql); c.close()
+    s = Store(db)
+    p,r = s.record('dblp','conf/www/Fall95','https://example.org/fall','Same paper in two events',
+                    'Abstract\x00with\x01text',['A Person'],{'arxiv':'2501.00001'})
+    s.record_tags(p,r,'dblp',[{'namespace':'dblp:booktitle','value':'WWW (Fall)'}])
+    s.decision(p,r,'Legacy',1995,'main','conf/www/Fall95','https://example.org/accept','accepted','Accept')
+    s.conn.execute("UPDATE venues SET acronym='WWW',name='WWW'")
+    meta={'snapshot_sha256':XML_SHA,'proceedings_key':'conf/www/1995f','bibliographic_year':1995,
+          'bibliography':{'booktitle':['WWW (Fall)'],'url':['db/conf/www/www1995f.html#Fall95']}}
+    s.conn.execute('UPDATE source_records SET metadata_json=?',(json.dumps(meta),));s.conn.commit();s.conn.close()
+    return p,r
+
+
+def add(store, external, event=None, status='accepted', venue='WWW', year=1995):
+    p,r = store.record('fixture',external,'https://example.org/'+external,'Same paper in two events',
+                       'Abstract\x00with\x01text',['A Person'],{'arxiv':'2501.00001'})
+    edition = store.decision(p,r,venue,year,'main',external,'https://example.org/'+status,status,status,event_key=event)
+    return p,r,edition
+
+
+def file(root, name, rows, event=None):
+    p=root/(name+'.jsonl');p.write_text(''.join(json.dumps(r)+'\n'for r in rows))
+    manifest={'source':'fixture','venue':'WWW','year':1995,'track':'main','count':len(rows),'list_url':'https://example.org/'+name}
+    if event is not None:manifest['event_key']=event
+    p.with_suffix('.manifest.json').write_text(json.dumps(manifest));return p
+
+
+class EventIdentityTests(unittest.TestCase):
+    def setUp(self):
+        t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);self.db=self.root/'sample.sqlite'
+
+    def test_same_paper_distinct_events_keep_both_attempts_and_decisions(self):
+        s=Store(self.db);self.addCleanup(s.conn.close)
+        paper,_,spring=add(s,'spring','www-1995-spring','rejected');_,_,fall=add(s,'fall','www-1995-fall')
+        s.conn.commit();self.assertNotEqual(spring,fall)
+        with redirect_stdout(io.StringIO())as out:querying.main(['--db',str(self.db),'--paper',str(paper)])
+        row=json.loads(out.getvalue())[0];self.assertEqual({a['event_key']for a in row['submissions']},{'www-1995-spring','www-1995-fall'})
+        self.assertEqual([a['event_key']for a in row['known_acceptances']],['www-1995-fall'])
+        self.assertEqual({a['latest_status']for a in row['submissions']},{'rejected','accepted'})
+        with redirect_stdout(io.StringIO())as out:querying.main(['--db',str(self.db),'--venue','WWW','--event-key','www-1995-spring','--status','rejected'])
+        self.assertEqual(len(json.loads(out.getvalue())),1)
+
+    def test_explicit_migration_preserves_ids_history_and_source_bytes(self):
+        paper,record=legacy(self.db);proposal=plan(self.db);self.assertFalse(proposal['before']['event_column'])
+        result=upgrade(proposal,self.db);self.assertEqual(result['updated_editions'],1)
+        self.assertEqual(result['after']['protected'],proposal['before']['protected'])
+        self.assertEqual(result['after']['editions_sha256'],proposal['before']['editions_sha256'])
+        self.assertEqual(upgrade(proposal,self.db)['status'],'already_applied')
+        s=Store(self.db);self.addCleanup(s.conn.close)
+        self.assertEqual(s.conn.execute('SELECT id,event_key FROM editions').fetchall(),[(1,'www-1995-fall')])
+        # An old file without event_key still has its original source anchor.
+        self.assertEqual(s.decision(paper,record,'WWW',1995,'main','conf/www/Fall95','https://example.org/accept','accepted','Accept'),1)
+        self.assertEqual(s.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
+        self.assertEqual(s.conn.execute('PRAGMA foreign_key_check').fetchall(),[])
+
+    def test_unverified_legacy_volume_is_not_automatically_labelled(self):
+        legacy(self.db)
+        with sqlite3.connect(self.db)as c:c.execute("UPDATE source_records SET metadata_json='{}'")
+        with self.assertRaisesRegex(ValueError,'complete pinned Fall'):plan(self.db)
+        with sqlite3.connect(self.db)as c:self.assertFalse(has_event_key(c))
+
+    def test_stale_plan_rolls_back_before_schema_change(self):
+        legacy(self.db);proposal=plan(self.db)
+        with sqlite3.connect(self.db)as c:c.execute("UPDATE editions SET track='unspecified'")
+        with self.assertRaisesRegex(ValueError,'stale'):upgrade(proposal,self.db)
+        with sqlite3.connect(self.db)as c:self.assertFalse(has_event_key(c));self.assertEqual(c.execute('SELECT track FROM editions').fetchone()[0],'unspecified')
+
+    def test_unscoped_source_and_wrong_refresh_event_fail(self):
+        s=Store(self.db);self.addCleanup(s.conn.close);_,r,e=add(s,'first','www-1995-fall');s.conn.commit()
+        p,r2=s.record('fixture','unscoped','https://example.org/u','Other work',None,[])
+        with self.assertRaisesRegex(ValueError,'event_key'):s.decision(p,r2,'WWW',1995,'main','unscoped','https://example.org/u','accepted','Accept')
+        with self.assertRaisesRegex(ValueError,'Cannot move'):s.decision(p,r,'WWW',1995,'main','first','https://example.org/u','accepted','Accept',event_key='www-1995-spring')
+        self.assertEqual(s.conn.execute('SELECT count(*) FROM editions').fetchone()[0],1)
+
+    def test_pinned_fall_cannot_claim_spring(self):
+        p,r=legacy(self.db);upgrade(plan(self.db),self.db);s=Store(self.db);self.addCleanup(s.conn.close)
+        with self.assertRaisesRegex(ValueError,'pinned WWW Fall'):s.decision(p,r,'WWW',1995,'main','conf/www/Fall95','https://example.org/u','accepted','Accept',event_key='www-1995-spring')
+
+    def test_manifest_event_mismatch_rolls_back_the_whole_file(self):
+        s=Store(self.db);s.conn.close()
+        base={'source':'fixture','venue':'WWW','year':1995,'track':'main','status':'accepted','title':'Input work','url':'https://example.org/p','authors':[]}
+        path=file(self.root,'bad',[dict(base,external_id='a',event_key='www-1995-spring'),dict(base,external_id='b',event_key='www-1995-fall')],'www-1995-spring')
+        with self.assertRaisesRegex(ValueError,'edition differs'):ingest_file(self.db,path)
+        with sqlite3.connect(self.db)as c:
+            self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],0)
+            self.assertEqual(c.execute('SELECT count(*) FROM editions').fetchone()[0],0)
+
+    def test_backfill_selects_the_explicit_event_and_preserves_attempts(self):
+        s=Store(self.db);add(s,'spring','www-1995-spring','rejected');add(s,'fall','www-1995-fall');s.conn.commit();s.conn.close()
+        row={'source':'fixture','external_id':'fall','venue':'WWW','year':1995,'track':'main','event_key':'www-1995-fall','status':'accepted','identifiers':{'openreview':'forumFall'}}
+        path=file(self.root,'backfill',[row],'www-1995-fall')
+        with redirect_stdout(io.StringIO()):result=backfill(self.db,[path])
+        self.assertEqual(result['submissions_after'],2)
+        with sqlite3.connect(self.db)as c:
+            self.assertEqual(c.execute("SELECT e.event_key FROM submission_identifiers si JOIN submissions s ON s.id=si.submission_id JOIN editions e ON e.id=s.edition_id WHERE si.scheme='openreview' AND si.value='forumFall'").fetchone()[0],'www-1995-fall')
+
+    def test_exact_paged_counts_cover_composite_key_boundaries(self):
+        s=Store(self.db);self.addCleanup(s.conn.close)
+        for i in range(7):add(s,'entry'+str(i),'www-1995-fall')
+        s.conn.commit();events=[];counts=exact_counts(s.conn,page_size=2,progress=events.append)
+        self.assertEqual(counts['source_records'],7)
+        self.assertEqual(counts['identifiers'],8)
+        self.assertEqual(counts['submission_sources'],7)
+        self.assertEqual([e['rows_counted']for e in events if e['table']=='identifiers'and e['complete']],[8])
+
+    def test_legacy_single_event_writes_and_reads_remain_available(self):
+        legacy(self.db);s=Store(self.db);self.addCleanup(s.conn.close)
+        add(s,'ordinary',venue='COLM',year=2025);s.conn.commit()
+        with redirect_stdout(io.StringIO())as out:querying.main(['--db',str(self.db),'--venue','COLM'])
+        self.assertEqual(json.loads(out.getvalue())[0]['known_acceptances'][-1]['event_key'],'')
+        with self.assertRaisesRegex(ValueError,'migration required'):
+            querying.main(['--db',str(self.db),'--event-key','www-1995-fall'])
+
+    def test_unexpected_extra_edition_column_requires_review(self):
+        legacy(self.db)
+        with sqlite3.connect(self.db)as c:c.execute('ALTER TABLE editions ADD COLUMN custom_data TEXT')
+        with self.assertRaisesRegex(ValueError,'Unexpected edition schema'):plan(self.db)
+
+    def test_file_spanning_two_events_without_manifest_key_rolls_back(self):
+        Store(self.db).conn.close()
+        base={'source':'fixture','venue':'WWW','year':1995,'track':'main','status':'accepted','title':'Input work','url':'https://example.org/p','authors':[]}
+        path=file(self.root,'two',[dict(base,external_id='a',event_key='www-1995-spring'),dict(base,external_id='b',event_key='www-1995-fall')])
+        with self.assertRaisesRegex(ValueError,'exactly one'):ingest_file(self.db,path)
+        with sqlite3.connect(self.db)as c:self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],0)
+
+    def test_cutover_sqlite_backup_remains_protected(self):
+        legacy(self.db);proposal=plan(self.db);Path(str(self.db.resolve())+'.postgresql-cutover.json').write_text('{}')
+        with self.assertRaisesRegex(RuntimeError,'cut over'):upgrade(proposal,self.db)
+
+
+@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service')
+class PostgreSQLEventIdentityTests(unittest.TestCase):
+    def setUp(self):
+        self.service=os.environ['PAPER_ATLAS_TEST_PG_SERVICE'];self.schema='pa_event_'+uuid.uuid4().hex[:12]
+        t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);self.db=self.root/'seed.sqlite';legacy(self.db)
+        self.addCleanup(self.cleanup)
+        with redirect_stdout(io.StringIO()):migrate(self.db,self.service,self.root/'migrate.json',schema=self.schema,min_free_gib=0)
+
+    def cleanup(self):
+        with connect(self.service)as c:
+            for (name,)in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(self.schema,)).fetchall():c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
+
+    def test_native_upgrade_preserves_ids_and_codec_then_accepts_separate_event(self):
+        proposal=plan(pg_service=self.service,pg_schema=self.schema)
+        result=upgrade(proposal,pg_service=self.service,pg_schema=self.schema)
+        self.assertEqual(result['after']['protected'],proposal['before']['protected'])
+        self.assertEqual(upgrade(proposal,pg_service=self.service,pg_schema=self.schema)['status'],'already_applied')
+        with opened_store(pg_service=self.service,pg_schema=self.schema)as s:
+            paper,_,spring=add(s,'spring','www-1995-spring','rejected');s.conn.commit()
+            self.assertNotEqual(spring,1)
+            self.assertEqual(s.conn.execute('SELECT abstract FROM papers WHERE id=?',(paper,)).fetchone()[0],'Abstract\x00with\x01text')
+        with redirect_stdout(io.StringIO())as out:querying.main(['--pg-service',self.service,'--pg-schema',self.schema,'--paper',str(paper)])
+        row=json.loads(out.getvalue())[0];self.assertEqual(len(row['submissions']),2);self.assertEqual(len(row['known_acceptances']),1)
+
+    def test_native_paged_counts_preserve_encoded_composite_key_order(self):
+        with opened_store(pg_service=self.service,pg_schema=self.schema)as s:
+            for value in ['a\x00b','a\x01b','a\x02b','z']:
+                s.conn.execute('INSERT INTO identifiers(scheme,value,paper_id) VALUES(?,?,?)',('custom',value,1))
+            s.conn.commit();counts=exact_counts(s.conn,page_size=2)
+            self.assertEqual(counts['identifiers'],6)
+            self.assertEqual(counts['submissions'],1)
+
+    def test_stale_native_plan_is_rejected_and_lock_released(self):
+        proposal=plan(pg_service=self.service,pg_schema=self.schema)
+        with opened_store(pg_service=self.service,pg_schema=self.schema)as s:s.conn.execute("UPDATE editions SET track='unspecified'");s.conn.commit()
+        with self.assertRaisesRegex(ValueError,'stale'):upgrade(proposal,pg_service=self.service,pg_schema=self.schema)
+        with opened_store(pg_service=self.service,pg_schema=self.schema)as s:self.assertFalse(has_event_key(s.conn))

tests/test_historical_predecessors.py

@@ -0,0 +1,151 @@
+"""Original series, exact edition boundaries, and complete candidate discovery."""
+from contextlib import closing
+from copy import deepcopy
+import gzip
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from paper_atlas.collectors.dblp import catalogue, XML_SHA
+from paper_atlas.collectors.dblp_records import collect, normalize_row
+from paper_atlas.ingestion.records import ingest_file
+
+FIXTURE = json.loads((Path(__file__).parent/'fixtures/historical_predecessors.json').read_text())
+EXAMPLES = {row['key']: row for row in FIXTURE['volumes']}
+
+
+def example(key, *, bibliographic_year=None):
+    fixed = EXAMPLES[key]
+    family = key.split('/')[1]
+    year = int(key.split('/')[-1])
+    volume = {'record_type': 'proceedings', 'key': key, 'fields': deepcopy(fixed['fields'])}
+    row = {'record_type': 'inproceedings', 'key': f'conf/{family}/SyntheticExample{year}', 'publtype': None,
+           'fields': {'title': [f'Synthetic historical research example {family} {year}'],
+                      'author': ['Jane Doe 0001', 'John Smith'], 'booktitle': fixed['fields']['booktitle'],
+                      'year': [str(bibliographic_year or year)], 'crossref': [key],
+                      'ee': [f'https://doi.org/10.1234/synthetic.{family}.{year}']}}
+    return row, volume
+
+
+class HistoricalPredecessorTests(unittest.TestCase):
+    def normalize(self, key, **kwargs):
+        row, volume = example(key, **kwargs)
+        fixed = EXAMPLES[key]
+        result, reason = normalize_row(row, {key: volume}, {fixed['expected_target_family']})
+        self.assertIsNone(reason)
+        self.assertIsNotNone(result)
+        return result
+
+    def test_ewsl_primary_series_retains_original_name_and_family_alias(self):
+        for year in [1987, 1988, 1989, 1991]:
+            with self.subTest(year=year):
+                record = self.normalize(f'conf/ecml/{year}')
+                self.assertEqual((record['venue'],record['venue_aliases'],record['year'],record['track']), ('EWSL',['ECML-PKDD'],year,'main'))
+                proof = record['metadata']['historical_predecessor_evidence']
+                self.assertEqual(proof['original_series'], 'EWSL')
+                self.assertEqual(proof['history_url'], 'https://ecmlpkdd2011.org/pastConferences.php')
+                self.assertEqual(record['metadata']['bibliography']['booktitle'], ['EWSL'])
+                self.assertEqual(record['metadata']['author_names_raw'], ['Jane Doe 0001','John Smith'])
+                self.assertEqual(record['authors'], ['Jane Doe','John Smith'])
+        self.assertEqual(EXAMPLES['conf/ecml/1989']['bibliographic_counts'], [])
+
+    def test_eps_ecp_ewsp_keep_original_series_instead_of_renaming_to_icaps(self):
+        for key in ['conf/aips/1990','conf/ecp/1997','conf/ecp/1999','conf/ewsp/1991']:
+            with self.subTest(key=key):
+                record = self.normalize(key)
+                fixed = EXAMPLES[key]
+                self.assertEqual(record['venue'], fixed['expected_original_series'])
+                self.assertEqual(record['venue_aliases'], ['ICAPS'])
+                self.assertEqual(record['track'], 'main')
+                proof = record['metadata']['historical_predecessor_evidence']
+                self.assertEqual(proof['history_url'], 'https://www.icaps-conference.org/conference-series/')
+                self.assertEqual(proof['proceedings_title'], fixed['fields']['title'][0])
+                self.assertEqual(proof['proceedings_year_fields'], fixed['fields']['year'])
+
+    def test_altered_titles_keys_suffixes_publication_years_and_unreviewed_editions_stay_held(self):
+        for key in ['conf/ecml/1988','conf/aips/1990','conf/ecp/1997','conf/ewsp/1991']:
+            row, volume = example(key)
+            for changed in ['title','key','suffix','publication_year']:
+                with self.subTest(key=key, changed=changed):
+                    r, v = deepcopy(row), deepcopy(volume)
+                    if changed == 'title': v['fields']['title'][0] += ', Affiliated Workshop'
+                    elif changed == 'key': v['key'] += 'w'; r['fields']['crossref'] = [v['key']]
+                    elif changed == 'suffix': r['fields']['booktitle'][0] += ' (Workshop)'
+                    else: v['fields']['year'] = ['2001']
+                    result, _ = normalize_row(r, {v['key']: v}, {EXAMPLES[key]['expected_target_family']})
+                    self.assertIsNone(result)
+        row, volume = example('conf/ecp/1997')
+        row['fields']['year'] = ['2001']; row['fields']['crossref'] = ['conf/ecp/2001']
+        volume['key'] = 'conf/ecp/2001'; volume['fields']['year'] = ['2001']
+        self.assertIsNone(normalize_row(row, {volume['key']:volume}, {'ICAPS'})[0])
+
+    def test_ecp1999_event_and_bibliographic_years_are_independently_retained(self):
+        record = self.normalize('conf/ecp/1999', bibliographic_year=2000)
+        self.assertEqual(record['year'], 1999)
+        self.assertEqual(record['metadata']['bibliographic_year'], 2000)
+        self.assertEqual(record['metadata']['bibliography']['year'], ['2000'])
+        self.assertEqual(record['metadata']['historical_predecessor_evidence']['proceedings_year_fields'], ['2000'])
+        self.assertEqual(record['metadata']['historical_predecessor_evidence']['event_year'], 1999)
+
+    def test_old_candidates_require_new_families_only_when_requested_years_need_them(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary); row, volume = example('conf/aips/1990')
+            db = root/'dblp-catalogue.sqlite'
+            with closing(sqlite3.connect(db)) as connection, connection:
+                connection.execute('CREATE TABLE proceedings(key TEXT PRIMARY KEY,metadata_json TEXT)')
+                connection.execute('INSERT INTO proceedings VALUES(?,?)', (volume['key'],json.dumps(volume)))
+            candidate = root/'candidate.jsonl'; candidate.write_text(json.dumps(row)+'\n')
+            db.with_name('dblp-ai-candidates.manifest.json').write_text(json.dumps({
+                'snapshot_sha256': XML_SHA, 'candidate_sha256': hashlib.sha256(candidate.read_bytes()).hexdigest(),
+                'candidate_records': 1, 'candidate_family_counts': {'aips':1},
+                'record_types': {'proceedings':1,'inproceedings':1}}))
+            # Tiny CI fixtures may live on tmpfs; production reserve is tested separately.
+            with patch('paper_atlas.collectors.dblp_records.shutil.disk_usage', return_value=SimpleNamespace(free=16*1024**3)):
+                result = collect(db, root/'early', ['ICAPS'], candidates=candidate, through_year=1990)
+            self.assertEqual((result['normalized_records'],result['files'][0]['venue']), (1,'EPS'))
+            for year, missing in [(1991,'ICAPS/ewsp'),(1997,'ICAPS/ecp')]:
+                out = root/str(year); out.mkdir(); sentinel=out/'previous.txt';sentinel.write_text('accepted previous output')
+                with self.subTest(year=year), self.assertRaisesRegex(ValueError, missing):
+                    collect(db, out, ['ICAPS'], candidates=candidate, through_year=year)
+                self.assertEqual(sentinel.read_text(), 'accepted previous output')
+                self.assertFalse((out/'dblp-normalization.json').exists())
+
+    def test_new_catalogue_includes_previously_missing_families_without_decision_inference(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary);dtd=root/'dblp.dtd';dtd.write_bytes(b'<!ENTITY eacute "&#233;">')
+            archive=root/'dblp.xml.gz'
+            body=''.join(f'<inproceedings key="conf/{family}/Synthetic"><title>Synthetic example</title><year>{year}</year><booktitle>{book}</booktitle><crossref>conf/{family}/{year}</crossref></inproceedings>' for family,year,book in [('ecp',1997,'ECP'),('ewsp',1991,'EWSP')])
+            with gzip.open(archive,'wb') as stream:stream.write(('<?xml version="1.0"?><!DOCTYPE dblp SYSTEM "dblp.dtd"><dblp>'+body+'</dblp>').encode())
+            result=catalogue(archive,dtd,root/'out',hashlib.sha256(archive.read_bytes()).hexdigest(),hashlib.md5(dtd.read_bytes()).hexdigest())
+            rows=[json.loads(line)for line in Path(result['file']).read_text().splitlines()]
+            self.assertEqual({r['key'] for r in rows}, {'conf/ecp/Synthetic','conf/ewsp/Synthetic'})
+            self.assertEqual(result['candidate_records'], 2)
+            self.assertTrue(all('status' not in r and 'venue' not in r for r in rows))
+
+    def test_native_import_keeps_original_series_aliases_and_full_source_evidence(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root=Path(temporary);db=root/'sample.sqlite'
+            for key in ['conf/ecml/1988','conf/ecp/1999']:
+                record=self.normalize(key);path=root/(key.replace('/','-')+'.jsonl')
+                path.write_text(json.dumps(record,ensure_ascii=False)+'\n')
+                path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','venue':record['venue'],'year':record['year'],'track':'main','count':1,'list_url':'https://doi.org/10.4230/dblp.xml.2026-11-01','file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'status':'partial','missing_abstract_count':1}))
+                ingest_file(db,path)
+            with closing(sqlite3.connect(db)) as connection:
+                rows=connection.execute('SELECT v.acronym,a.alias,e.year FROM editions e JOIN venues v ON v.id=e.venue_id JOIN venue_aliases a ON a.venue_id=v.id ORDER BY v.acronym').fetchall()
+                self.assertEqual(rows,[('ECP','ICAPS',1999),('EWSL','ECML-PKDD',1988)])
+                metadata=[json.loads(r[0])for r in connection.execute('SELECT metadata_json FROM source_records')]
+                self.assertEqual({r['historical_predecessor_evidence']['original_series'] for r in metadata}, {'ECP','EWSL'})
+                self.assertEqual(connection.execute('SELECT count(*) FROM decisions WHERE status=?',('accepted',)).fetchone()[0],2)
+                self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(),[])
+
+    def test_existing_aips_normalization_remains_original_aips(self):
+        row, volume=example('conf/aips/1990');row['fields']['booktitle']=['AIPS'];row['fields']['year']=['1994'];row['fields']['crossref']=['conf/aips/1994']
+        volume['key']='conf/aips/1994';volume['fields']['title']=['AIPS 1994 Proceedings'];volume['fields']['year']=['1994']
+        record,reason=normalize_row(row,{volume['key']:volume},{'ICAPS'})
+        self.assertIsNone(reason);self.assertEqual((record['venue'],record['venue_aliases']),('AIPS',['ICAPS']))
+        self.assertNotIn('historical_predecessor_evidence',record['metadata'])

tests/test_historical_primary_series.py

@@ -0,0 +1,115 @@
+"""Regression coverage for primary series once named Workshop, with strict scope boundaries."""
+from contextlib import closing
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+
+from paper_atlas.collectors.dblp_records import normalize_row
+from paper_atlas.collectors.dblp_scope import scope
+from paper_atlas.ingestion.records import ingest_file
+
+ICML_HISTORY = 'https://icml.cc/Conferences/2008/past_icmls.shtml.html'
+IROS_HISTORY = 'https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/'
+
+
+class HistoricalPrimarySeriesTests(unittest.TestCase):
+    def example(self, family, year, title, book=None, key=None):
+        key = key or f'conf/{family}/{year}'
+        book = book or ('ML' if family == 'icml' else 'IROS')
+        row = {'record_type':'inproceedings', 'key':f'conf/{family}/Paper{year}', 'publtype':None,
+               'fields':{'title':['An independently identified historical research paper'],
+                         'author':['Jane Doe'], 'booktitle':[book], 'year':[str(year)],
+                         'crossref':[key], 'ee':[f'https://doi.org/10.1234/{family}.{year}']}}
+        volume = {'record_type':'proceedings', 'key':key,
+                  'fields':{'title':[title], 'year':[str(year)]}}
+        return row, volume
+
+    def test_icml_workshop_primary_editions_are_not_silently_filtered(self):
+        examples = {
+            1989:'Proceedings of the Sixth International Workshop on Machine Learning (ML 1989), Cornell University, Ithaca, New York, USA, June 26-27, 1989',
+            1991:'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA',
+            1992:'Proceedings of the Ninth International Workshop on Machine Learning (ML 1992), Aberdeen, Scotland, UK, July 1-3, 1992',
+        }
+        for year, title in examples.items():
+            with self.subTest(year=year):
+                row, volume = self.example('icml', year, title)
+                record, reason = normalize_row(row, {volume['key']:volume}, {'ICML'})
+                self.assertIsNone(reason)
+                self.assertEqual((record['venue'],record['year'],record['track'],record['status']), ('ICML',year,'main','accepted'))
+                self.assertEqual(record['metadata']['proceedings_title'], title)
+                self.assertEqual(record['metadata']['track_scope_evidence_url'], ICML_HISTORY)
+                self.assertEqual(record['metadata']['bibliographic_year'], year)
+
+    def test_iros_first_three_editions_are_primary_series(self):
+        examples = {
+            1988:"IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988",
+            1989:"IEEE International Workshop on Intelligent Robots and Systems '89, Proceedings. IROS 1989, September 4-6, 1989, Tsukuba, Japan",
+            1990:"IEEE International Workshop on Intelligent Robots and Systems '90, Towards a New Frontier of Applications, Proceedings. IROS 1990, July 3-6, 1990, Ibaraki, Japan",
+        }
+        for year, title in examples.items():
+            with self.subTest(year=year):
+                row, volume = self.example('iros', year, title)
+                record, reason = normalize_row(row, {volume['key']:volume}, {'IROS'})
+                self.assertIsNone(reason)
+                self.assertEqual((record['venue'],record['year'],record['track']), ('IROS',year,'main'))
+                self.assertEqual(record['metadata']['proceedings_title'], title)
+                self.assertEqual(record['metadata']['track_scope_evidence_url'], IROS_HISTORY)
+
+    def test_main_named_icml_predecessors_also_carry_primary_scope_provenance(self):
+        examples = {
+            1988:'Machine Learning, Proceedings of the Fifth International Conference on Machine Learning, Ann Arbor, Michigan, USA, June 12-14, 1988',
+            1990:'Machine Learning, Proceedings of the Seventh International Conference on Machine Learning, Austin, Texas, USA, June 21-23, 1990',
+        }
+        for year, title in examples.items():
+            row, volume = self.example('icml', year, title)
+            edition, reason = scope(row, volume)
+            self.assertIsNone(reason)
+            self.assertEqual(edition['track'], 'main')
+            self.assertEqual(edition['track_scope_evidence_url'], ICML_HISTORY)
+
+    def test_wrong_keys_titles_suffixes_and_unknown_years_keep_workshop_boundary(self):
+        title = 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA'
+        for year, changed, book, key in [
+            (1991,title,'ML','conf/icml/1991w'),
+            (1991,title,'ML (Workshops)','conf/icml/1991'),
+            (1991,title.replace('Eighth','Ninth'),'ML','conf/icml/1991'),
+            (1991,title + ', Affiliated Workshop','ML','conf/icml/1991'),
+            (1987,'Machine Learning Workshop 1987, Irvine','ML','conf/icml/1987'),
+        ]:
+            with self.subTest(year=year, book=book, key=key, title=changed):
+                row, volume = self.example('icml',year,changed,book,key)
+                record, reason = normalize_row(row,{volume['key']:volume},{'ICML'})
+                self.assertIsNone(record)
+                self.assertEqual(reason,'separate_workshop')
+                self.assertNotIn('track_scope_evidence_url',scope(row,volume)[0])
+        title = "IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988"
+        for changed, book, key in [
+            (title,'IROS','conf/iros/1988w'),
+            (title,'IROS (Workshops)','conf/iros/1988'),
+            (title.replace('Tokyo','Another city'),'IROS','conf/iros/1988'),
+        ]:
+            row, volume = self.example('iros',1988,changed,book,key)
+            self.assertEqual(normalize_row(row,{volume['key']:volume},{'IROS'})[1],'separate_workshop')
+
+    def test_primary_scope_proof_survives_native_file_ingestion(self):
+        title = 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA'
+        row, volume = self.example('icml',1991,title)
+        record, reason = normalize_row(row,{volume['key']:volume},{'ICML'})
+        self.assertIsNone(reason)
+        with tempfile.TemporaryDirectory() as temporary:
+            root=Path(temporary);path=root/'records.jsonl';path.write_text(json.dumps(record)+'\n')
+            path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','count':1,'list_url':ICML_HISTORY,'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest()}))
+            ingest_file(root/'test.sqlite',path)
+            with closing(sqlite3.connect(root/'test.sqlite')) as connection:
+                self.assertEqual(connection.execute('SELECT track FROM editions').fetchone()[0],'main')
+                self.assertEqual(connection.execute('SELECT status FROM decisions').fetchone()[0],'accepted')
+                metadata=json.loads(connection.execute('SELECT metadata_json FROM source_records').fetchone()[0])
+                self.assertEqual(metadata['proceedings_title'],title)
+                self.assertEqual(metadata['track_scope_evidence_url'],ICML_HISTORY)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_histories.py

@@ -0,0 +1,114 @@
+from pathlib import Path
+import tempfile
+import unittest
+
+from paper_atlas.audits.histories import report
+from paper_atlas.db.store import Store
+
+
+def add_history_fixture(store):
+    def event(work, forum, venue, year, track, status, observed):
+        paper, record = store.record('history_fixture', work + '/' + forum,
+            'https://example.org/' + forum, work + ' title\x00 preserved', 'Abstract', ['Jane Doe'],
+            {'arxiv': work})
+        store.decision(paper, record, venue, year, track, forum,
+            'https://example.org/' + forum + '/' + str(status), status, status or '',
+            identities=[('openreview', forum)], observed_at=observed)
+        return paper
+
+    # The negative observation was collected later: observation order is not chronology.
+    event('2501.00001', 'HistoryA01', 'ICLR', 2024, 'main', 'rejected', '2026-02-01T00:00:00Z')
+    event('2501.00001', 'HistoryA02', 'ICML', 2025, 'main', 'accepted', '2026-01-01T00:00:00Z')
+    event('2501.00002', 'HistoryB01', 'ICLR', 2024, 'main', 'rejected', '2026-01-01T00:00:00Z')
+    event('2501.00002', 'HistoryB01', 'ICLR', 2024, 'main', 'accepted', '2026-02-01T00:00:00Z')
+    event('2501.00003', 'HistoryC01', 'ICLR', 2024, 'main', 'desk_rejected', '2026-01-01T00:00:00Z')
+    event('2501.00003', 'HistoryC02', 'ICLR', 2024, 'workshop_example', 'accepted', '2026-02-01T00:00:00Z')
+    event('2501.00004', 'HistoryD01', 'ICML', 2025, 'main', 'accepted', '2026-01-01T00:00:00Z')
+    event('2501.00004', 'HistoryD01', 'ICML', 2025, 'main', 'withdrawn', '2026-02-01T00:00:00Z')
+    event('2501.00004', 'HistoryD02', 'ICLR', 2024, 'main', 'rejected', '2026-01-01T00:00:00Z')
+    event('2501.00005', 'HistoryE01', 'COLM', 2026, 'main', None, '2026-01-01T00:00:00Z')
+    store.conn.commit()
+
+
+class HistoryTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.db = Path(self.tmp.name) / 'sample.sqlite'
+        self.store = Store(self.db)
+        self.addCleanup(self.store.conn.close)
+
+    def test_distinct_attempts_venues_and_latest_acceptances_are_not_observation_counts(self):
+        add_history_fixture(self.store)
+        result = report(self.db)
+        self.assertEqual(result['counts'], {
+            'papers_with_submissions': 5, 'papers_with_multiple_submissions': 3,
+            'papers_with_historical_negative': 4, 'papers_with_current_known_acceptance': 3,
+            'papers_with_negative_and_acceptance_in_distinct_attempts': 2,
+            'papers_with_negative_and_acceptance_in_distinct_venues': 1,
+            'submissions': 8, 'decision_observations': 9,
+            'attempts_with_historical_negative': 5,
+            'attempts_with_historical_negative_and_acceptance': 2,
+        })
+        self.assertEqual(result['latest_attempt_status_counts'],
+                         {'accepted': 3, 'desk_rejected': 1, 'rejected': 2, 'unknown': 1, 'withdrawn': 1})
+        self.assertEqual(len(result['examples']), 2)
+        first = result['examples'][0]
+        self.assertIn('\x00', first['title'])
+        self.assertEqual({r['venue'] for r in first['attempts']}, {'ICLR', 'ICML'})
+        self.assertTrue(all(r['decisions'][0]['sources'] for r in first['attempts']))
+        self.assertTrue(all(r['decisions'][0]['decision_date'] is None for r in first['attempts']))
+        self.assertIn('Not inferred', first['chronology'])
+        self.assertEqual(result['status'], 'observed_history_audit_not_final_dedup')
+        self.assertTrue(all(v == 0 for v in result['evidence_relationship_checks'].values()))
+
+    def test_missing_and_mislinked_source_evidence_is_reported_without_repair(self):
+        add_history_fixture(self.store)
+        self.store.conn.execute('DELETE FROM decision_sources WHERE decision_id=1')
+        self.store.conn.execute('DELETE FROM submission_sources WHERE submission_id=1')
+        # Record3 belongs to paper2; move it to paper1 to create a real mismatch.
+        self.store.conn.execute('UPDATE source_records SET paper_id=1 WHERE id=3')
+        self.store.conn.commit()
+        result = report(self.db, examples=0)
+        checks = result['evidence_relationship_checks']
+        self.assertEqual(checks['decisions_without_source_links'], 1)
+        self.assertEqual(checks['submissions_without_source_links'], 1)
+        self.assertEqual(checks['submission_source_paper_mismatches'], 1)
+        self.assertEqual(checks['decision_source_paper_mismatches'], 2)
+        self.assertEqual(self.store.conn.execute('SELECT paper_id FROM source_records WHERE id=3').fetchone()[0], 1)
+
+    def test_example_limit_does_not_change_global_counts(self):
+        add_history_fixture(self.store)
+        none, one = report(self.db, examples=0), report(self.db, examples=1)
+        self.assertEqual(none['counts'], one['counts'])
+        self.assertEqual(none['examples'], [])
+        self.assertEqual(len(one['examples']), 1)
+
+    def test_empty_snapshot_and_invalid_limit_do_not_create_missing_database(self):
+        empty = report(self.db)
+        self.assertTrue(all(v == 0 for v in empty['counts'].values()))
+        self.assertEqual(empty['latest_attempt_status_counts'], {})
+        absent = self.db.parent / 'absent.sqlite'
+        with self.assertRaisesRegex(ValueError, 'Examples'):
+            report(absent, examples=-1)
+        with self.assertRaisesRegex(ValueError, 'Existing database'):
+            report(absent)
+        self.assertFalse(absent.exists())
+
+    def test_truncated_example_keeps_the_cross_venue_acceptance_visible(self):
+        paper, record = self.store.record('fixture', 'many', 'https://example.org/many',
+                                          'Many attempts', 'Abstract', [])
+        for index in range(26):
+            accepted = index == 25
+            forum = f'BoundedForum{index}'
+            self.store.decision(paper, record, 'ICML' if accepted else 'ICLR', 2025, 'main', forum,
+                                'https://example.org/' + forum, 'accepted' if accepted else 'rejected',
+                                'Accept' if accepted else 'Reject', identities=[('openreview', forum)])
+        self.store.conn.commit()
+        result = report(self.db)
+        example, = result['examples']
+        self.assertEqual((example['attempt_count'], len(example['attempts'])), (26, 20))
+        self.assertTrue(example['attempts_truncated'])
+        self.assertEqual({row['latest_observed_status'] for row in example['attempts']},
+                         {'accepted', 'rejected'})
+        self.assertEqual(result['counts']['submissions'], 26)

tests/test_iclr_archive.py

@@ -0,0 +1,148 @@
+"""Historical accepted lists, repeated presentations and preserved work histories."""
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors import iclr_archive as archive
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.records import ingest_file
+
+OBSERVED = '2026-11-09T00:00:00Z'
+
+
+def entry(identifier, title='A shared research paper', display='Jane Doe and Bob Smith'):
+    return f'<li><div><a href="http://arxiv.org/abs/{identifier}">{title}</a>, {display}</div></li>'
+
+
+def page2015():
+    return ('<h3>Main Conference - Oral Presentations</h3><ol>'+entry('1412.0001v2')+'</ol>'
+            '<h3>Main Conference - Poster Presentations</h3><ol>'+entry('1412.0001v2')+'</ol>'
+            '<h3>Workshop Papers</h3><ol>'+entry('1412.0002', 'A workshop paper')+'</ol>')
+
+
+class IclrArchiveTests(unittest.TestCase):
+    def frozen(self, root, year, html):
+        root.mkdir()
+        data = html.encode()
+        (root/'page.html').write_bytes(data)
+        receipt = {'url': archive.PAGES[year][0], 'file': 'page.html', 'status': 200,
+                   'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'finished_at': OBSERVED}
+        (root/'receipts.json').write_text(json.dumps({'receipts': [receipt]}))
+        return root
+
+    def test_oral_poster_repeat_is_one_attempt_with_all_source_variants(self):
+        rows = archive.parse_page(page2015(), 2015, {'observed_at': OBSERVED})
+        main, workshop = rows
+        self.assertEqual((main['track'], workshop['track']), ('main', 'workshop'))
+        self.assertEqual(main['identifiers'], {'arxiv': '1412.0001'})
+        self.assertEqual(main['external_id'], '2015/main/1412.0001')
+        self.assertEqual(main['metadata']['presentations'], ['oral', 'poster'])
+        self.assertEqual(len(main['metadata']['source_variants']), 2)
+        self.assertEqual(main['metadata']['source_variants'][0]['version'], 'v2')
+        self.assertEqual(main['authors'], ['Jane Doe', 'Bob Smith'])
+        self.assertTrue(all(r['status'] == 'accepted' and r['abstract'] is None for r in rows))
+
+    def test_2014_author_span_outside_paragraph_and_trailing_separator(self):
+        html = ('<h3>Conference Proceedings</h3><p>conference papers accepted</p>'
+                '<div><b><a href="http://arxiv.org/abs/1312.0001">A <i>research</i> title</a></b></div>'
+                '<p><span style="font-style: italic">Jane Doe; </span>'
+                '<span style="font-style:italic">Bob Smith; Carol Wu;</span></p>'
+                '<div><a href="http://arxiv.org/abs/1312.0002">Another title</a>'
+                '<div><i><span>First Author; </span><span>Second Author</span></i></div></div>')
+        row, unwrapped = archive.parse_page(html, 2014, {'observed_at': OBSERVED})
+        self.assertEqual(row['title'], 'A research title')
+        self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith', 'Carol Wu'])
+        self.assertEqual(row['metadata']['source_variants'][0]['authors_display'], 'Jane Doe; Bob Smith; Carol Wu;')
+        self.assertEqual(row['track'], 'main')
+        self.assertEqual(unwrapped['authors'], ['First Author', 'Second Author'])
+
+    def test_same_arxiv_in_different_tracks_keeps_separate_source_attempts(self):
+        html = page2015().replace('1412.0002', '1412.0001v3')
+        rows = archive.parse_page(html, 2015, {'observed_at': OBSERVED})
+        self.assertEqual(len(rows), 2)
+        self.assertEqual({r['external_id'] for r in rows}, {'2015/main/1412.0001', '2015/workshop/1412.0001'})
+        self.assertEqual(rows[1]['metadata']['source_variants'][0]['version'], 'v3')
+
+    def test_2016_code_links_and_breaks_do_not_become_authors(self):
+        html = ('<h1>Accepted Papers (Conference Track)</h1><h3>Oral Presentations</h3><ol>'
+                '<li><a href="http://arxiv.org/abs/1511.00001">A title</a>'
+                '<a href="https://example.org/code">[code]</a><br>Jane Doe, Bob Smith</li></ol>'
+                '<h3>Poster Presentations</h3>')
+        row, = archive.parse_page(html, 2016, {'observed_at': OBSERVED})
+        self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith'])
+        self.assertEqual(row['metadata']['presentations'], ['oral'])
+
+    def test_conflicting_repeat_unknown_sections_and_unparsed_links_fail(self):
+        variants = [page2015().replace('Poster Presentations', 'Other Presentations'),
+                    page2015().replace('</ol>', '</ol><a href="http://arxiv.org/abs/1412.0003">stray</a>', 1),
+                    page2015().replace('Jane Doe and Bob Smith', 'Different Author', 1)]
+        for html in variants:
+            with self.subTest(html=html), self.assertRaises(ValueError):
+                archive.parse_page(html, 2015, {'observed_at': OBSERVED})
+
+    def test_arxiv_versions_are_preserved_and_unsafe_links_rejected(self):
+        self.assertEqual(archive.arxiv_link('https://arxiv.org/pdf/1511.00001v3.pdf'), ('1511.00001', 'v3'))
+        self.assertIsNone(archive.arxiv_link('https://arxiv.org.example.org/abs/1511.00001'))
+        for url in ['https://arxiv.org/abs/1511.00001?other=1', 'ftp://arxiv.org/abs/1511.00001',
+                    'https://user@arxiv.org/abs/1511.00001', 'https://arxiv.org/abs/not-a-paper']:
+            with self.subTest(url=url), self.assertRaises(ValueError):
+                archive.arxiv_link(url)
+
+    def test_receipt_hash_status_path_and_symlink_cannot_escape_snapshot(self):
+        for failure in ['hash', 'status', 'path', 'symlink']:
+            with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary:
+                root = Path(temporary)
+                source = self.frozen(root/'source', 2015, page2015())
+                file = source/'receipts.json'
+                data = json.loads(file.read_text())
+                receipt = data['receipts'][0]
+                if failure == 'hash': receipt['sha256'] = '0'*64
+                elif failure == 'status': receipt['status'] = 404
+                elif failure == 'path': receipt['file'] = '../outside.html'
+                else:
+                    (root/'outside.html').write_text(page2015())
+                    (source/'page.html').unlink()
+                    (source/'page.html').symlink_to(root/'outside.html')
+                file.write_text(json.dumps(data))
+                with self.assertRaises(ValueError):
+                    archive.read_page(2015, snapshots=source)
+
+    def test_changed_inventory_leaves_no_completed_output(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            source = self.frozen(root/'source', 2015, page2015())
+            with self.assertRaises(ValueError):
+                archive.collect(2015, root/'out', snapshots=source)
+            self.assertFalse((root/'out/iclr-archive-2015.json').exists())
+
+    def test_native_import_keeps_rejected_to_accepted_history_and_abstract(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            source = self.frozen(root/'source', 2015, page2015())
+            with patch.dict(archive.PAGES, {2015: (archive.PAGES[2015][0], {'main': 1, 'workshop': 1})}):
+                result = archive.collect(2015, root/'out', snapshots=source)
+                with self.assertRaises(FileExistsError):
+                    archive.collect(2015, root/'out', snapshots=source)
+            db = root/'sample.sqlite'
+            store = Store(db)
+            pid, rid = store.record('arxiv', '1412.0001', 'https://arxiv.org/abs/1412.0001',
+                                    'A shared research paper', 'Keep this canonical abstract.',
+                                    ['Jane Doe', 'Bob Smith'], {'arxiv': '1412.0001'}, {})
+            store.decision(pid, rid, 'Earlier Conference', 2014, 'main', 'original-attempt',
+                           'https://example.org/rejected', 'rejected', 'explicit earlier decision', observed_at=OBSERVED)
+            store.conn.commit()
+            store.conn.close()
+            for manifest in result['files']:
+                ingest_file(db, manifest['file'])
+                ingest_file(db, manifest['file'])
+            with sqlite3.connect(db) as c:
+                self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 2)
+                self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (pid,)).fetchone()[0], 'Keep this canonical abstract.')
+                history = c.execute('SELECT v.acronym,e.year,d.status FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year', (pid,)).fetchall()
+                self.assertEqual(history, [('Earlier Conference', 2014, 'rejected'), ('ICLR', 2015, 'accepted')])
+                self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 3)
+                self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])

tests/test_identifier_curation.py

@@ -0,0 +1,185 @@
+import copy
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+from paper_atlas.curation.identifiers import SCOPE, digest, json_bytes, prepare
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.records import ingest_file
+
+
+class IdentifierCurationTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.output = self.root / 'prepared'
+        self.row = {'source':'dblp','external_id':'conf/uai/example','url':'https://example.org/paper',
+                    'title':'A sufficiently long example research paper title','abstract':None,
+                    'authors':['Jane Doe','John Smith'],'venue':'UAI','year':2001,'track':'main',
+                    'status':'accepted','tags':[{'namespace':'dblp:topic','value':'version\u2029study\x00'}],
+                    'metadata':{'pages':'203-210','raw_title':'Original\u2029title'}}
+        self.evidence = self.root / 'evidence.json'
+        self.evidence.write_bytes(json_bytes({'url':'https://example.org/archive','comment':'Conference version'}))
+        self.path = self.root / 'records.jsonl'
+        self.raw = json_bytes(self.row)
+        self.path.write_bytes(self.raw)
+        self.manifest = {'source':'dblp','venue':'UAI','year':2001,'track':'main','count':1,
+                         'list_url':'https://example.org/list','file_sha256':digest(self.raw),
+                         'status':'partial','missing_abstract_count':1,'parser_version':2}
+        self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
+        self.plan = {'format_version':1,'inputs':[self.input(self.path)],'assertions':[
+            {'source':'dblp','external_id':self.row['external_id'],'record_sha256':digest(self.raw),
+             'identifiers':{'arxiv':'1408.1485'},'evidence':[{
+                 'file':str(self.evidence),'sha256':digest(self.evidence.read_bytes()),
+                 'url':'https://example.org/archive','checked_at':'2026-11-08T08:00:00Z',
+                 'note':'Reviewed title, authors, publication and version evidence'}]}]}
+
+    def input(self, path):
+        return {'file':str(path),'sha256':digest(path.read_bytes()),
+                'manifest_sha256':digest(path.with_suffix('.manifest.json').read_bytes())}
+
+    def run_plan(self, plan=None):
+        path = self.root / 'plan.json'
+        path.write_bytes(json_bytes(self.plan if plan is None else plan))
+        return prepare(path, self.output)
+
+    def test_only_identifiers_and_provenance_change_and_raw_inputs_survive(self):
+        result = self.run_plan()
+        self.assertEqual(self.path.read_bytes(), self.raw)
+        prepared = json.loads((self.output / self.path.name).read_bytes())
+        provenance = prepared['metadata'].pop('identifier_curation')
+        self.assertEqual(prepared.pop('identifiers'), {'arxiv':'1408.1485'})
+        self.assertEqual(prepared, self.row)
+        self.assertEqual(provenance[0]['record_sha256'], digest(self.raw))
+        self.assertEqual((self.output / provenance[0]['evidence'][0]['file']).read_bytes(), self.evidence.read_bytes())
+        manifest = json.loads((self.output / 'records.manifest.json').read_bytes())
+        for key, value in self.manifest.items():
+            if key != 'file_sha256': self.assertEqual(manifest[key], value)
+        self.assertEqual(manifest['file_sha256'], digest((self.output / self.path.name).read_bytes()))
+        self.assertEqual(result['assertions_applied'], 1)
+        self.assertEqual(result['identifiers_added'], 1)
+        self.assertEqual(result['verification_scope'], SCOPE)
+        self.assertFalse(result['semantic_identity_verified'])
+        self.assertFalse(result['database_written'])
+        self.assertEqual(list(self.output.glob('*.jsonl')), [self.output / self.path.name])
+        with self.assertRaisesRegex(ValueError, 'already exist'): self.run_plan()
+
+    def test_late_invalid_file_publishes_nothing(self):
+        second = self.root / 'second.jsonl'
+        second.write_bytes(json_bytes({**self.row,'external_id':'second'}))
+        manifest = {**self.manifest,'file_sha256':digest(second.read_bytes()),'count':2}
+        second.with_suffix('.manifest.json').write_bytes(json_bytes(manifest))
+        self.plan['inputs'].append(self.input(second))
+        with self.assertRaisesRegex(ValueError, 'count mismatch'): self.run_plan()
+        self.assertFalse(self.output.exists())
+        self.assertFalse(list(self.root.glob('.identifier-curation-*')))
+        self.assertEqual(self.path.read_bytes(), self.raw)
+
+    def test_unasserted_rows_keep_exact_physical_bytes_and_doi_is_additive(self):
+        untouched = (json.dumps({**self.row,'external_id':'unselected'},ensure_ascii=False,indent=None) + '  ').encode()
+        self.path.write_bytes(self.raw + untouched)
+        self.path.with_suffix('.manifest.json').write_bytes(json_bytes({**self.manifest,
+            'file_sha256':digest(self.path.read_bytes()),'count':2,'missing_abstract_count':2}))
+        self.plan['inputs'] = [self.input(self.path)]
+        self.plan['assertions'][0]['identifiers']['doi'] = '10.1234/example'
+        result = self.run_plan()
+        with (self.output / self.path.name).open('rb') as stream:
+            first = json.loads(next(stream))
+            self.assertEqual(next(stream), untouched)
+        self.assertEqual(first['identifiers'], {'arxiv':'1408.1485','doi':'10.1234/example'})
+        self.assertEqual(result['identifiers_added'], 2)
+
+    def test_missing_duplicate_and_mismatched_assertions_fail_closed(self):
+        for mode in ('missing','duplicate','line_hash','duplicate_input'):
+            with self.subTest(mode=mode):
+                plan = copy.deepcopy(self.plan)
+                assertion = plan['assertions'][0]
+                if mode == 'missing': assertion['external_id'] = 'absent'
+                elif mode == 'duplicate': plan['assertions'].append(copy.deepcopy(assertion))
+                elif mode == 'line_hash': assertion['record_sha256'] = '0' * 64
+                else:
+                    other = self.root / 'duplicate.jsonl'
+                    other.write_bytes(self.raw)
+                    other.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
+                    plan['inputs'].append(self.input(other))
+                with self.assertRaises(ValueError): self.run_plan(plan)
+                self.assertFalse(self.output.exists())
+
+    def test_identifier_conflict_does_not_overwrite_and_same_value_is_auditable(self):
+        self.row['identifiers'] = {'arxiv':'cs/0307069'}
+        self.path.write_bytes(json_bytes(self.row))
+        self.manifest['file_sha256'] = digest(self.path.read_bytes())
+        self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
+        self.plan['inputs'] = [self.input(self.path)]
+        self.plan['assertions'][0]['record_sha256'] = digest(self.path.read_bytes())
+        with self.assertRaisesRegex(ValueError, 'identifier conflict'): self.run_plan()
+        self.assertFalse(self.output.exists())
+        self.plan['assertions'][0]['identifiers']['arxiv'] = 'cs/0307069'
+        self.assertEqual(self.run_plan()['identifiers_added'], 0)
+
+    def test_unsupported_changes_identifiers_and_unbound_evidence_are_rejected(self):
+        for mode in ('title','bad_month','version','unknown_scheme','doi_case','evidence_hash','credentials','timestamp'):
+            with self.subTest(mode=mode):
+                plan = copy.deepcopy(self.plan)
+                assertion = plan['assertions'][0]
+                evidence = assertion['evidence'][0]
+                if mode == 'title': assertion['title'] = 'Rewrite forbidden'
+                elif mode == 'bad_month': assertion['identifiers'] = {'arxiv':'1413.12345'}
+                elif mode == 'version': assertion['identifiers'] = {'arxiv':'1408.1485v2'}
+                elif mode == 'unknown_scheme': assertion['identifiers'] = {'url':'https://example.org'}
+                elif mode == 'doi_case': assertion['identifiers'] = {'doi':'10.1234/UPPER'}
+                elif mode == 'evidence_hash': evidence['sha256'] = '0' * 64
+                elif mode == 'credentials': evidence['url'] = 'https://user:secret@example.org'
+                else: evidence['checked_at'] = '2026-11-08T08:00:00'
+                with self.assertRaises(ValueError): self.run_plan(plan)
+                self.assertFalse(self.output.exists())
+
+    def test_input_and_manifest_hashes_and_duplicate_json_keys_are_checked(self):
+        for mode in ('input','manifest','duplicate_json','invalid_evidence_json'):
+            with self.subTest(mode=mode):
+                plan = copy.deepcopy(self.plan)
+                if mode == 'input':
+                    plan['inputs'][0]['sha256'] = '0' * 64
+                elif mode == 'manifest':
+                    plan['inputs'][0]['manifest_sha256'] = '0' * 64
+                elif mode == 'duplicate_json':
+                    raw = self.raw[:-2] + b',"source":"dblp"}\n'
+                    self.path.write_bytes(raw)
+                    self.path.with_suffix('.manifest.json').write_bytes(json_bytes({**self.manifest,'file_sha256':digest(raw)}))
+                    plan['inputs'] = [self.input(self.path)]
+                else:
+                    self.evidence.write_bytes(b'not json')
+                    plan['assertions'][0]['evidence'][0]['sha256'] = digest(self.evidence.read_bytes())
+                with self.assertRaises(ValueError): self.run_plan(plan)
+                self.assertFalse(self.output.exists())
+                self.path.write_bytes(self.raw)
+                self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
+
+    def test_curated_anchor_import_does_not_merge_extended_version_or_its_history(self):
+        db = self.root / 'papers.sqlite'
+        store = Store(db)
+        try:
+            original, _ = store.record('arxiv','1408.1485','https://example.org/one',self.row['title'],None,self.row['authors'],{'arxiv':'1408.1485'})
+            extended, record = store.record('arxiv','cs/0307069','https://example.org/two',self.row['title'],None,self.row['authors'],{'arxiv':'cs/0307069'})
+            store.decision(extended,record,'ICLR',2025,'main','another-attempt','https://example.org/reject','rejected','Reject')
+            store.conn.commit()
+        finally:
+            store.conn.close()
+        self.run_plan()
+        ingest_file(db,self.output / self.path.name)
+        store = Store(db)
+        try:
+            c = store.conn
+            self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 2)
+            self.assertEqual(c.execute('SELECT paper_id FROM source_records WHERE external_id=?',(self.row['external_id'],)).fetchone()[0], original)
+            self.assertEqual(c.execute('SELECT s.paper_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE v.acronym=?',('ICLR',)).fetchone()[0], extended)
+            self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0], 0)
+            self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
+        finally:
+            store.conn.close()
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_incremental_coverage.py

@@ -0,0 +1,116 @@
+from contextlib import redirect_stdout
+import hashlib
+import io
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+
+from paper_atlas.ingestion.records import ingest_file
+
+
+class IncrementalCoverageTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.db = self.root / 'papers.sqlite'
+
+    def file(self, name, external_ids, **options):
+        rows = [{'source':'example','external_id':key,'url':'https://example.org/'+key,
+                 'venue':'Example','year':2026,'track':'main','status':'accepted',
+                 'title':'Independent research contribution '+key,'abstract':'Known abstract',
+                 'authors':[], 'identifiers':{'doi':'10.1234/'+key}} for key in external_ids]
+        path = self.root / (name+'.jsonl')
+        path.write_bytes((''.join(json.dumps(row)+'\n' for row in rows)).encode())
+        manifest = {key:rows[0][key] for key in ('source','venue','year','track')}
+        manifest.update(list_url='https://example.org/2026', count=len(rows),status='partial',
+                        file_sha256=hashlib.sha256(path.read_bytes()).hexdigest())
+        manifest.update(options)
+        path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
+        return path
+
+    def ingest(self, path):
+        with redirect_stdout(io.StringIO()):
+            ingest_file(self.db,path)
+
+    def incremental(self, previous=2, prior_expected=2, expected=4):
+        return {'coverage_mode':'incremental','coverage_previous_count':previous,
+                'coverage_previous_expected_count':prior_expected,'coverage_expected_count':expected}
+
+    def coverage(self):
+        with sqlite3.connect(self.db) as c:
+            return c.execute('SELECT status,expected_count,imported_count FROM edition_sources').fetchall()
+
+    def test_append_and_refresh_count_actual_source_records_without_erasing_earlier_batch(self):
+        self.ingest(self.file('first',['a','b']))
+        self.ingest(self.file('append',['c'],**self.incremental()))
+        self.assertEqual(self.coverage(),[('partial',4,3)])
+        self.ingest(self.file('refresh',['c'],**self.incremental(3,4,4)))
+        self.assertEqual(self.coverage(),[('partial',4,3)])
+        with sqlite3.connect(self.db) as c:
+            self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],3)
+            self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],3)
+            runs = c.execute('SELECT records_written,checkpoint_json FROM ingest_runs ORDER BY id').fetchall()
+            self.assertEqual([r[0] for r in runs],[2,1,1])
+            self.assertEqual(json.loads(runs[-1][1])['coverage'],
+                             {'mode':'incremental','previous_count':3,'expected_count':4,'imported_count':3})
+            self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
+
+    def test_stale_parent_stops_before_source_writes(self):
+        self.ingest(self.file('first',['a','b']))
+        with self.assertRaisesRegex(ValueError,'parent changed'):
+            self.ingest(self.file('stale',['c'],**self.incremental(1,2,4)))
+        self.assertEqual(self.coverage(),[('partial',2,2)])
+        with sqlite3.connect(self.db) as c:
+            self.assertEqual(c.execute("SELECT count(*) FROM source_records WHERE external_id='c'").fetchone()[0],0)
+            self.assertEqual(c.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
+
+    def test_late_source_bound_failure_rolls_back_new_records_and_coverage(self):
+        self.ingest(self.file('first',['a','b']))
+        with self.assertRaisesRegex(ValueError,'source bounds'):
+            self.ingest(self.file('overflow',['c'],**self.incremental(2,2,2)))
+        self.assertEqual(self.coverage(),[('partial',2,2)])
+        with sqlite3.connect(self.db) as c:
+            for table in ('papers','source_records','submissions','decisions'):
+                self.assertEqual(c.execute('SELECT count(*) FROM '+table).fetchone()[0],2)
+
+    def test_multiple_listings_cannot_be_silently_aggregated(self):
+        self.ingest(self.file('one',['a']))
+        self.ingest(self.file('two',['b'],list_url='https://example.org/another-list'))
+        before=self.coverage()
+        with self.assertRaisesRegex(ValueError,'unambiguous source listing'):
+            self.ingest(self.file('ambiguous',['c'],**self.incremental(2,1,4)))
+        self.assertEqual(self.coverage(),before)
+
+    def test_untracked_source_rows_require_reconciliation(self):
+        self.ingest(self.file('first',['a','b']))
+        with sqlite3.connect(self.db) as c:
+            c.execute('UPDATE edition_sources SET imported_count=1')
+        with self.assertRaisesRegex(ValueError,'actual source records'):
+            self.ingest(self.file('stale-summary',['c'],**self.incremental(1,2,4)))
+        self.assertEqual(self.coverage(),[('partial',2,1)])
+
+    def test_first_partial_chunk_can_declare_an_independently_known_source_total(self):
+        self.ingest(self.file('first',['a'],**self.incremental(0,None,4)))
+        self.assertEqual(self.coverage(),[('partial',4,1)])
+
+    def test_complete_listing_is_not_demoted_by_a_partial_chunk(self):
+        self.ingest(self.file('complete',['a','b'],status='complete'))
+        with self.assertRaisesRegex(ValueError,'not partial'):
+            self.ingest(self.file('append',['c'],**self.incremental()))
+        self.assertEqual(self.coverage(),[('complete',2,2)])
+
+    def test_invalid_mode_counts_and_missing_scope_fail_before_database_creation(self):
+        for options in ({'coverage_mode':'unknown'}, self.incremental(True,2,4),
+                        self.incremental(2,5,4),self.incremental(-1,2,4),
+                        {'coverage_mode':'incremental'},
+                        {**self.incremental(),'status':'complete'}):
+            with self.subTest(options=options):
+                with self.assertRaises(ValueError):self.ingest(self.file('invalid',['a'],**options))
+                self.assertFalse(self.db.exists())
+
+
+if __name__=='__main__':
+    unittest.main()

tests/test_index_core.py

@@ -53,6 +53,50 @@ class StoreTests(unittest.TestCase):
         self.assertNotEqual(other,paper)
         self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
 
+    def test_identifier_anchor_does_not_absorb_a_same_title_arxiv_version(self):
+        conference,first=self.record('arxiv','1408.1485')
+        extended,second=self.record('arxiv','cs/0307069')
+        self.store.decision(extended,second,'ICLR',2025,'main','separate','https://example.org/reject','rejected','Reject')
+        resolved,_=self.record('dblp','conf/uai/example',{'arxiv':'1408.1485'})
+        self.assertEqual(resolved,conference);self.assertNotEqual(conference,extended)
+        c=self.store.conn
+        self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
+        self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],extended)
+        self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0)
+        self.assertEqual(c.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
+        # A refresh of the anchored original source is equally unambiguous.
+        self.assertEqual(self.record('arxiv','1408.1485')[0],conference)
+        self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
+
+    def test_ambiguous_title_aborts_whole_file_without_merging_existing_entities(self):
+        self.record('arxiv','1408.1485');self.record('arxiv','cs/0307069')
+        self.store.conn.commit()
+        rows=[{'source':'dblp','external_id':'conf/uai/new','url':'https://example.org/new',
+               'title':'An unrelated earlier record in this file','abstract':None,'authors':[],
+               'venue':'UAI','year':2001,'track':'main','status':'accepted'},
+              {'source':'dblp','external_id':'conf/uai/ambiguous','url':'https://example.org/ambiguous',
+               'title':'A sufficiently long example research paper title','abstract':None,
+               'authors':['Jane Doe','John Smith'],'identifiers':{'doi':'10.0000/not-yet-known'},
+               'venue':'UAI','year':2001,'track':'main','status':'accepted'}]
+        path=Path(self.tmp.name)/'ambiguous.jsonl'
+        path.write_text(''.join(json.dumps(row)+'\n' for row in rows))
+        path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','list_url':'https://example.org/list','count':2}))
+        with self.assertRaisesRegex(ValueError,'Ambiguous paper identity'):ingest_file(self.db,path)
+        c=self.store.conn
+        for table,count in [('papers',2),('source_records',2),('submissions',0),('decisions',0),('paper_merges',0)]:
+            self.assertEqual(c.execute('SELECT count(*) FROM '+table).fetchone()[0],count,table)
+        self.assertEqual(c.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed')
+
+    def test_explicit_bridge_merges_only_its_identifier_matches(self):
+        first,_=self.record('arxiv','2601.00001',authors=['First Author'])
+        second,_=self.record('arxiv','2601.00002',{'doi':'10.0000/bridge'},authors=['Second Author'])
+        unrelated,_=self.record('arxiv','2601.00003')
+        resolved,_=self.record('official','bridge',{'arxiv':'2601.00001','doi':'10.0000/bridge'})
+        self.assertEqual(resolved,min(first,second))
+        self.assertEqual(self.store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],2)
+        self.assertIsNotNone(self.store.conn.execute('SELECT 1 FROM papers WHERE id=?',(unrelated,)).fetchone())
+        self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
+
     def test_manifest_count_mismatch_rolls_back_records(self):
         path=Path(self.tmp.name)/'source.jsonl'
         path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'})+'\n')

tests/test_joint_editions.py

@@ -0,0 +1,187 @@
+from contextlib import closing, redirect_stdout
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+import uuid
+
+from paper_atlas.collectors.dblp import XML_SHA
+from paper_atlas.collectors.dblp_scope import scope
+from paper_atlas.collectors.dblp_records import normalize_row
+from paper_atlas.db.store import Store
+from paper_atlas.db.backend import open_store
+from paper_atlas.ingestion.joint_editions import (TARGET, MARKER, inspect, apply_plan, run)
+from paper_atlas.postgres.client import connect
+from paper_atlas.postgres.migration import migrate
+from paper_atlas.postgres.schema import quote
+from paper_atlas import querying
+
+
+def seed(store):
+    c = store.conn
+    c.execute("INSERT INTO venues(acronym,name,field,venue_type) VALUES(?,?,'reasoning','conference')", (TARGET,TARGET))
+    target = c.execute('SELECT id FROM venues WHERE acronym=?',(TARGET,)).fetchone()[0]
+    for alias in ['ECAI','IJCAI']:
+        c.execute('INSERT INTO venue_aliases VALUES(?,?)',(target,alias))
+    for year in [2018,2019,2022]:
+        metadata = {'snapshot_sha256':XML_SHA,'proceedings_key':'conf/ijcai/'+str(year),
+                    'bibliographic_year':year,'bibliography':{'booktitle':['IJCAI']}}
+        p,r=store.record('dblp','conf/ijcai/fixture'+str(year),'https://dblp.org/rec/fixture',
+                         'Original publication '+str(year),'Abstract\x00with\x01data',[],
+                         {'doi':'10.24963/ijcai.'+str(year)+'/1'}, {})
+        store.record_tags(p,r,'dblp',[{'namespace':'dblp:booktitle','value':'IJCAI'}])
+        store.decision(p,r,'IJCAI',year,'main','fixture'+str(year),'https://example.org/decision','accepted','listed')
+        stored=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(r,)).fetchone()[0]);stored.update(metadata)
+        c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(stored),r))
+    c.commit()
+
+
+def plan(c):
+    return {**inspect(c),'database':{'engine':'sqlite','path':'fixture'}}
+
+
+class JointEditionTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp=tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
+        self.db=Path(self.tmp.name)/'sample.sqlite'
+        self.store=Store(self.db); self.addCleanup(self.store.conn.close)
+        seed(self.store); self.c=self.store.conn
+
+    def test_apply_preserves_every_protected_row_and_edition_id(self):
+        before=plan(self.c); ids=[r['id']for r in before['editions']]
+        result=apply_plan(self.c,before);self.c.commit()
+        self.assertEqual(result['updated_editions'],2)
+        self.assertEqual(result['content_fingerprints'],before['content_fingerprints'])
+        self.assertEqual([r['id']for r in result['editions']],ids)
+        untouched=self.c.execute('SELECT v.acronym FROM editions e JOIN venues v ON v.id=e.venue_id WHERE e.year=2019').fetchone()[0]
+        self.assertEqual(untouched,'IJCAI')
+        self.assertEqual(self.c.execute('PRAGMA foreign_key_check').fetchall(),[])
+        self.assertEqual(self.c.execute('SELECT count(*) FROM schema_migrations WHERE name=?',(MARKER,)).fetchone()[0],1)
+        with redirect_stdout(io.StringIO())as out:
+            querying.main(['--db',str(self.db),'--venue','ECAI'])
+        self.assertEqual(len(json.loads(out.getvalue())),2)
+        self.assertEqual(apply_plan(self.c,before)['updated_editions'],0)
+
+    def test_refresh_of_old_normalized_source_does_not_recreate_ordinary_edition(self):
+        before=plan(self.c);apply_plan(self.c,before)
+        for year in [2018,2022]:
+            row=self.c.execute('''SELECT r.paper_id,r.id,d.observed_at FROM source_records r
+                JOIN submissions s ON s.source_record_id=r.id JOIN decisions d ON d.submission_id=s.id
+                WHERE r.external_id=?''',('conf/ijcai/fixture'+str(year),)).fetchone()
+            self.store.decision(row[0],row[1],'IJCAI',year,'main','fixture'+str(year),
+                                'https://example.org/decision','accepted','listed',observed_at=row[2])
+        self.assertEqual(inspect(self.c)['content_fingerprints'],before['content_fingerprints'])
+        self.assertEqual(self.c.execute('SELECT count(*) FROM editions').fetchone()[0],3)
+
+    def test_any_changed_source_text_rejects_stale_plan_before_membership_updates(self):
+        before=plan(self.c)
+        self.c.execute("UPDATE source_records SET title='New evidence' WHERE external_id='conf/ijcai/fixture2018'")
+        with self.assertRaisesRegex(ValueError,'Stale repair plan'):
+            apply_plan(self.c,before)
+        self.assertEqual(inspect(self.c)['status'],'ready')
+
+    def test_conflicting_target_edition_is_never_merged_or_deleted(self):
+        target=self.c.execute('SELECT id FROM venues WHERE acronym=?',(TARGET,)).fetchone()[0]
+        self.c.execute("INSERT INTO editions(venue_id,year,track) VALUES(?,2018,'main')",(target,))
+        with self.assertRaisesRegex(ValueError,'colliding'):
+            inspect(self.c)
+        self.assertEqual(self.c.execute('SELECT count(*) FROM editions WHERE year=2018').fetchone()[0],2)
+
+    def test_wrong_source_volume_and_snapshot_fail_closed(self):
+        original=self.c.execute("SELECT metadata_json FROM source_records WHERE external_id='conf/ijcai/fixture2018'").fetchone()[0]
+        for key,value in [('proceedings_key','conf/ijcai/2018w'),('snapshot_sha256','wrong'),('bibliographic_year',2017),('bibliography',{'booktitle':['Other']})]:
+            m=json.loads(original);m[key]=value
+            self.c.execute("UPDATE source_records SET metadata_json=? WHERE external_id='conf/ijcai/fixture2018'",(json.dumps(m),))
+            with self.subTest(key=key),self.assertRaisesRegex(ValueError,'Source evidence'):
+                inspect(self.c)
+
+    def test_missing_source_support_and_wrong_aliases_are_not_guessed(self):
+        self.c.execute("DELETE FROM venue_aliases WHERE alias='ECAI'")
+        with self.assertRaisesRegex(ValueError,'aliases'):
+            inspect(self.c)
+        self.c.rollback()
+        self.c.execute('DELETE FROM submission_sources WHERE source_record_id=(SELECT id FROM source_records WHERE external_id=?)',('conf/ijcai/fixture2022',))
+        with self.assertRaisesRegex(ValueError,'Every submission'):
+            inspect(self.c)
+
+    def test_cli_plan_readonly_hash_binding_identity_and_repeat_apply(self):
+        out=Path(self.tmp.name)/'plan.json'
+        with redirect_stdout(io.StringIO()):run(self.db,out)
+        self.assertEqual(inspect(self.c)['status'],'ready')
+        raw=out.read_bytes();sha=hashlib.sha256(raw).hexdigest()
+        with patch('paper_atlas.ingestion.joint_editions.open_store')as writer:
+            with self.assertRaisesRegex(ValueError,'hash differs'):
+                run(self.db,Path(self.tmp.name)/'bad.json',plan=out,plan_sha256='0'*64)
+            writer.assert_not_called()
+        with redirect_stdout(io.StringIO()):
+            result=run(self.db,Path(self.tmp.name)/'applied.json',plan=out,plan_sha256=sha)
+            again=run(self.db,Path(self.tmp.name)/'retry.json',plan=out,plan_sha256=sha)
+        self.assertEqual(result['updated_editions'],2);self.assertEqual(again['updated_editions'],0)
+        self.assertEqual(result['plan_sha256'],sha);self.assertEqual(again['plan_sha256'],sha)
+        altered=json.loads(raw);altered['database']['path']='another.sqlite'
+        bad=Path(self.tmp.name)/'wrong-db.json';bad.write_text(json.dumps(altered))
+        with self.assertRaisesRegex(ValueError,'another database'):
+            run(self.db,Path(self.tmp.name)/'wrong-result.json',plan=bad,plan_sha256=hashlib.sha256(bad.read_bytes()).hexdigest())
+
+    def test_invalid_empty_plan_fails_before_opening_writer(self):
+        f=Path(self.tmp.name)/'empty.json';f.write_text('{}')
+        with patch('paper_atlas.ingestion.joint_editions.open_store')as writer:
+            with self.assertRaises(ValueError):
+                run(self.db,Path(self.tmp.name)/'empty-result.json',plan=f,plan_sha256=hashlib.sha256(f.read_bytes()).hexdigest())
+            writer.assert_not_called()
+
+    def test_collector_only_changes_the_two_exact_main_volume_keys(self):
+        for year in [2018,2019,2022]:
+            volume={'key':'conf/ijcai/'+str(year),'fields':{'title':['IJCAI '+str(year)],'year':[str(year)]}}
+            row={'record_type':'inproceedings','key':'conf/ijcai/One'+str(year),'fields':{'booktitle':['IJCAI'],
+                 'year':[str(year)],'crossref':[volume['key']],'title':['Original research title'],
+                 'author':['Jane Doe'],'ee':['https://doi.org/10.24963/ijcai.'+str(year)+'/1']}}
+            result,error=normalize_row(row,{volume['key']:volume},{'IJCAI','ECAI'})
+            self.assertIsNone(error)
+            self.assertEqual(result['venue'],TARGET if year in(2018,2022)else'IJCAI')
+            self.assertEqual(result['metadata']['bibliography']['booktitle'],['IJCAI'])
+            self.assertEqual(result['title'],'Original research title')
+            if year in(2018,2022):
+                self.assertIn('joint_edition_evidence',result['metadata'])
+                row['fields']['booktitle']=['IJCAI (Workshop)']
+                scoped,_=scope(row,volume)
+                self.assertEqual(scoped['venue'],'IJCAI')
+                self.assertEqual(scoped['track'],'workshop')
+
+
+@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL service')
+class PostgreSQLJointTests(unittest.TestCase):
+    def setUp(self):
+        self.service=os.environ['PAPER_ATLAS_TEST_PG_SERVICE'];self.target='pa_joint_'+uuid.uuid4().hex[:12]
+        self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
+        self.db=Path(self.tmp.name)/'sample.sqlite';s=Store(self.db);seed(s);s.conn.close()
+        self.addCleanup(self.cleanup)
+        with redirect_stdout(io.StringIO()):migrate(self.db,self.service,Path(self.tmp.name)/'migration.json',schema=self.target,min_free_gib=0)
+
+    def cleanup(self):
+        with connect(self.service)as c:
+            for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(self.target,)).fetchall():
+                c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
+
+    def test_native_repair_is_idempotent_keeps_codec_and_provenance(self):
+        p=Path(self.tmp.name)/'plan.json'
+        with redirect_stdout(io.StringIO()):
+            before=run(None,p,pg_service=self.service,pg_schema=self.target)
+            after=run(None,Path(self.tmp.name)/'applied.json',pg_service=self.service,pg_schema=self.target,plan=p,plan_sha256=hashlib.sha256(p.read_bytes()).hexdigest())
+        self.assertEqual(after['content_fingerprints'],before['content_fingerprints'])
+        with closing(open_store(pg_service=self.service,pg_schema=self.target).conn)as c:
+            self.assertEqual(apply_plan(c,before)['updated_editions'],0)
+            self.assertEqual(c.execute('SELECT abstract FROM papers ORDER BY id LIMIT 1').fetchone()[0],'Abstract\x00with\x01data')
+
+    def test_native_stale_source_plan_preserves_both_original_memberships(self):
+        with closing(open_store(pg_service=self.service,pg_schema=self.target).conn)as c:
+            before=plan(c)
+            c.execute("UPDATE source_records SET title='Changed' WHERE external_id=?",('conf/ijcai/fixture2022',));c.commit()
+            with self.assertRaisesRegex(ValueError,'Stale repair plan'):
+                apply_plan(c,before)
+            c.rollback()
+            self.assertEqual(inspect(c)['status'],'ready')

tests/test_postgres.py

@@ -175,6 +175,22 @@ class PostgreSQLIntegrationTests(unittest.TestCase):
         finally:
             reader.close()
 
+    def test_history_audit_matches_sqlite_for_actual_postgresql_snapshot(self):
+        from paper_atlas.audits.histories import report
+        from tests.test_histories import add_history_fixture
+        store = Store(self.db)
+        try:
+            add_history_fixture(store)
+        finally:
+            store.conn.close()
+        expected = report(self.db)
+        self.assertEqual(self.migrate()['status'], 'complete')
+        actual = report(pg_service=self.service, pg_schema=self.target)
+        self.assertEqual(actual['engine'], 'postgresql')
+        self.assertEqual(actual['schema'], self.target)
+        for key in ('counts', 'latest_attempt_status_counts', 'evidence_relationship_checks', 'examples'):
+            self.assertEqual(actual[key], expected[key], key)
+
     def interrupt_after_first_table(self):
         calls = 0
         def interrupt(*args):

tests/test_postgres_ingestion.py

@@ -87,6 +87,82 @@ class PostgreSQLIngestionTests(unittest.TestCase):
         with redirect_stdout(io.StringIO()):
             ingest_file(None, path, pg_service=self.service, pg_schema=self.schema)
 
+    def test_incremental_source_coverage_counts_prior_chunks_and_refreshes_on_postgres(self):
+        rows = [self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
+                for key in ('incremental1','incremental2','incremental3')]
+        listing = 'https://example.org/uai/2001'
+        def manifest(path, **options):
+            target=path.with_suffix('.manifest.json')
+            value=json.loads(target.read_text())
+            value.update(list_url=listing,status='partial',**options)
+            target.write_text(json.dumps(value))
+        first=self.file('initial',rows[:2]);manifest(first);self.ingest(first)
+        second=self.file('append',[rows[2]])
+        manifest(second,coverage_mode='incremental',coverage_previous_count=2,
+                 coverage_previous_expected_count=2,coverage_expected_count=4)
+        self.ingest(second)
+        manifest(second,coverage_mode='incremental',coverage_previous_count=3,
+                 coverage_previous_expected_count=4,coverage_expected_count=4)
+        self.ingest(second)
+        with closing(ReadConnection(self.service,self.schema)) as c:
+            coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
+            self.assertEqual(tuple(coverage.values),('partial',4,3))
+            stored=c.execute('SELECT abstract FROM source_records WHERE external_id=?',('incremental3',)).fetchone()
+            self.assertEqual(stored[0],rows[2]['abstract'])
+            self.assertEqual(c.execute("SELECT count(*) FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name='dblp'").fetchone()[0],3)
+            run=c.execute('SELECT records_written,checkpoint_json FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()
+            self.assertEqual(run[0],1)
+            self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
+
+    def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
+        listing='https://example.org/uai/2001'
+        rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
+              for key in ('bound1','bound2','bound3')]
+        first=self.file('initial',rows[:2]);target=first.with_suffix('.manifest.json')
+        value=json.loads(target.read_text());value.update(list_url=listing,status='partial');target.write_text(json.dumps(value))
+        self.ingest(first)
+        with closing(ReadConnection(self.service,self.schema)) as c:
+            before={table:c.execute('SELECT count(*) FROM '+table).fetchone()[0]
+                    for table in ('papers','source_records','submissions','decisions')}
+        path=self.file('overflow',[rows[2]]);target=path.with_suffix('.manifest.json')
+        value=json.loads(target.read_text());value.update(list_url=listing,status='partial',coverage_mode='incremental',
+            coverage_previous_count=2,coverage_previous_expected_count=2,coverage_expected_count=2)
+        target.write_text(json.dumps(value))
+        with self.assertRaisesRegex(ValueError,'source bounds'):self.ingest(path)
+        with closing(ReadConnection(self.service,self.schema)) as c:
+            self.assertEqual({table:c.execute('SELECT count(*) FROM '+table).fetchone()[0] for table in before},before)
+            coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
+            self.assertEqual(tuple(coverage.values),('partial',2,2))
+            self.assertEqual(c.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
+
+    def test_incremental_postgres_does_not_overwrite_a_concurrently_changed_parent(self):
+        listing='https://example.org/uai/2001'
+        rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
+              for key in ('concurrent1','concurrent2','concurrent3')]
+        first=self.file('initial',rows[:2]);target=first.with_suffix('.manifest.json')
+        value=json.loads(target.read_text());value.update(list_url=listing,status='partial');target.write_text(json.dumps(value))
+        self.ingest(first)
+        path=self.file('racing',[rows[2]]);target=path.with_suffix('.manifest.json')
+        value=json.loads(target.read_text());value.update(list_url=listing,status='partial',coverage_mode='incremental',
+            coverage_previous_count=2,coverage_previous_expected_count=2,coverage_expected_count=4)
+        target.write_text(json.dumps(value))
+        original=Store.record
+        def change_parent(store,*args,**kwargs):
+            result=original(store,*args,**kwargs)
+            # A separate committed SQL writer need not honor the native
+            # advisory lock. Its update must survive our rejected file.
+            with connect(self.service) as other:
+                other.execute('SET search_path TO '+quote(self.schema)+',pg_catalog')
+                other.execute('UPDATE edition_sources SET expected_count=99 WHERE list_url=%s',(listing,))
+            return result
+        with patch.object(Store,'record',change_parent):
+            with self.assertRaisesRegex(ValueError,'parent changed during import'):self.ingest(path)
+        with closing(ReadConnection(self.service,self.schema)) as c:
+            coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
+            self.assertEqual(tuple(coverage.values),('partial',99,2))
+            self.assertEqual(c.execute('SELECT count(*) FROM source_records WHERE external_id=?',('concurrent3',)).fetchone()[0],0)
+            self.assertEqual(c.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
+
     def test_native_writes_keep_nul_historical_ids_and_later_acceptance(self):
         store = self.store()
         try:
@@ -110,6 +186,34 @@ class PostgreSQLIngestionTests(unittest.TestCase):
         finally:
             reader.close()
 
+    def test_identity_anchor_keeps_other_arxiv_version_and_ambiguous_file_rolls_back(self):
+        title='An identical long research title shared by conference and extended versions'
+        authors=['Jane Doe','John Smith']
+        store=self.store()
+        try:
+            first,_=store.record('arxiv','1408.1485','https://arxiv.org/abs/1408.1485',title,'Conference abstract',authors)
+            second,_=store.record('arxiv','cs/0307069','https://arxiv.org/abs/cs/0307069',title,'Extended abstract',authors)
+            resolved,_=store.record('dblp','known','https://example.org/known',title,None,authors,{'arxiv':'1408.1485'})
+            self.assertEqual(resolved,first);self.assertNotEqual(first,second)
+            store.conn.commit()
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],3)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)  # migration fixture only
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
+        finally:store.conn.close()
+        rows=[self.row('official','earlier',status='accepted',venue='UAI',year=2001),
+              dict(self.row('official','ambiguous',status='accepted',venue='UAI',year=2001),
+                   title=title,abstract=None,authors=authors,identifiers={})]
+        with self.assertRaisesRegex(ValueError,'Ambiguous paper identity'):self.ingest(self.file('ambiguous-identity',rows))
+        store=self.store()
+        try:
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],3)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],4)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
+            self.assertEqual(store.conn.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
+            self.assertEqual(store.conn.execute("SELECT count(*) FROM source_records WHERE external_id IN (?,?)",('earlier','ambiguous')).fetchone()[0],0)
+        finally:store.conn.close()
+
     def test_file_import_is_idempotent_and_refreshes_only_own_tags(self):
         row = self.row(); row['tags'] = [{'namespace': 'openreview_snapshot:area', 'value': 'theory'}]
         path = self.file('reject', [row]); self.ingest(path)
@@ -243,6 +347,50 @@ class PostgreSQLIngestionTests(unittest.TestCase):
         finally:
             reader.close()
 
+    def test_coverage_matches_sqlite_with_history_aliases_and_missing_source_abstracts(self):
+        from paper_atlas.audits.coverage import report
+        from tests.test_coverage_report import add_coverage_evidence
+        registry = self.root / 'venues.json'
+        registry.write_text(json.dumps({'scope_note': 'Candidate scope', 'venues': [
+            {'acronym': 'coling', 'field': 'language'}, {'acronym': 'AAAI', 'field': 'general_ai'}]}))
+        sample = Store(self.root / 'seed.sqlite')
+        try:
+            add_coverage_evidence(sample)
+        finally:
+            sample.conn.close()
+        store = self.store()
+        try:
+            add_coverage_evidence(store)
+        finally:
+            store.conn.close()
+        sqlite_report = report(self.root / 'seed.sqlite', registry)
+        observed = []
+        def reader(*args, **kwargs):
+            connection = open_read_connection(*args, **kwargs)
+            observed.append((connection.execute('SHOW transaction_read_only').fetchone()[0],
+                             connection.execute('SHOW transaction_isolation').fetchone()[0]))
+            return connection
+        with patch('paper_atlas.audits.coverage.open_read_connection', side_effect=reader):
+            actual = report(None, registry, pg_service=self.service, pg_schema=self.schema)
+        self.assertEqual(observed, [('on', 'repeatable read')])
+        self.assertEqual(actual['engine'], 'postgresql')
+        self.assertEqual(actual['schema'], self.schema)
+        self.assertEqual(actual['pg_service'], self.service)
+        def content(result):
+            values = json.loads(json.dumps(result['venues']))
+            for venue in values:
+                for edition in venue['editions']:
+                    for key in ('id', 'venue_id'):
+                        edition.pop(key)
+            return values
+        self.assertEqual(content(actual), content(sqlite_report))
+        self.assertEqual([r['acronym'] for r in actual['imported_venues_outside_registry']], ['ICLR'])
+        edition = actual['venues'][0]['editions'][0]
+        self.assertEqual(edition['latest_observed_status_counts']['accepted'], 1)
+        self.assertEqual(edition['latest_observed_status_counts']['unknown'], 1)
+        self.assertEqual(edition['decision_observation_counts']['rejected'], 1)
+        self.assertEqual(edition['papers_without_abstract_text'], 1)
+
     def test_backfill_merges_evidence_without_reimporting_or_losing_history(self):
         files = []
         for source, external, title in [('official', 'legacy', 'First bibliographic name'), ('openreview_snapshot', 'forum', 'Renamed submission')]:

tests/test_public_openreview.py

@@ -45,10 +45,12 @@ class PublicOpenReviewTests(unittest.TestCase):
 
     def test_only_explicit_decisions_and_verified_scope_become_events(self):
         for value in ('Pending', 'Active', 'Submitted to NeurIPS 2025', 'TBD', '',
-                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject'):
+                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject',
+                      'Workshop', 'Poster Workshop', 'Oral Workshop'):
             self.assertIsNone(negative_status(value), value)
         self.assertEqual(negative_status('Reject'), 'rejected')
         self.assertEqual(negative_status('Desk rejected'), 'desk_rejected')
+        self.assertEqual(negative_status('Withdraw'), 'withdrawn')
         self.assertEqual(negative_status('NeurIPS 2023 Conference Withdrawn Submission'), 'withdrawn')
         for raw, expected in [('main', 'main'), ('Position', 'position'),
                               ('Datasets & Benchmarks', 'datasets_benchmarks')]:
@@ -98,6 +100,121 @@ class PublicOpenReviewTests(unittest.TestCase):
                 self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
                 self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
 
+    def test_missing_title_is_held_without_losing_valid_negative_records(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            result = self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [
+                self.row(id='SJUdkecgx', title=''),
+                self.row(id='NoTitle001', title=None),
+                self.row(id='Whitespace', title='  '),
+                self.row(id='ValidReject'),
+                self.row(id='Withdraw01', status='Withdraw'),
+                self.row(id='Workshop01', status='Workshop'),
+            ]}, families=['iclr'], years=[2017])
+            self.assertEqual(result['input_rows'], 6)
+            self.assertEqual(result['normalized_records'], 2)
+            self.assertEqual(result['source_files'][0]['excluded_or_held'],
+                             {'missing_title': 3, 'not_explicit_negative': 1})
+            held_path = Path(result['held_file'])
+            self.assertEqual(hashlib.sha256(held_path.read_bytes()).hexdigest(),
+                             result['held_file_sha256'])
+            held = [json.loads(line) for line in held_path.read_text().splitlines()]
+            self.assertEqual([row['id'] for row in held], ['SJUdkecgx', 'NoTitle001', 'Whitespace'])
+            self.assertTrue(all(row['reason'] == 'missing_title' and row['status'] == 'Reject'
+                                for row in held))
+            self.assertEqual([row['provenance']['source_line'] for row in held], [1, 2, 3])
+            self.assertNotIn('private@example.org', held_path.read_text())
+            self.assertNotIn('private review', held_path.read_text())
+            manifest = result['files'][0]
+            self.assertEqual((manifest['parser_version'], manifest['status']), (3, 'partial'))
+            rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
+            self.assertEqual([(row['identifiers']['openreview'], row['status']) for row in rows],
+                             [('ValidReject', 'rejected'), ('Withdraw01', 'withdrawn')])
+            for row in held:
+                self.assertEqual(row['provenance']['source_revision'], REVISION)
+
+    def test_nontext_title_still_fails_collection(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            with self.assertRaisesRegex(ValueError, 'lacks a title'):
+                self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [self.row(title=['invalid'])]},
+                             families=['iclr'], years=[2017])
+            self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
+            self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
+
+    def workshop_cases(self):
+        return [
+            (2024, 'workshop_me_fomo', 'ICLR 2024 Workshop MEFoMo Withdrawn Submission', 'withdrawn', 'ME-FoMo'),
+            (2024, 'workshop_genai4dm', 'ICLR 2024 Workshop GenAI4DM Withdrawn Submission', 'withdrawn', 'GenAI4DM'),
+            (2025, 'workshop_agenticai', 'ICLR 2025 Workshop AgenticAI Reject', 'rejected', 'AgenticAI'),
+            (2025, 'workshop_lmrl', 'ICLR 2025 Workshop LMRL Withdrawn Submission', 'withdrawn', 'LMRL'),
+            (2026, 'workshop_llm_reasoning', 'ICLR 2026 Workshop LLM Reasoning Withdrawn Submission', 'withdrawn', 'LLM_Reasoning'),
+        ]
+
+    def test_iclr_workshop_labels_require_opt_in_and_retain_identity_evidence(self):
+        for year, track, label, status, group in self.workshop_cases():
+            with self.subTest(label=label):
+                raw = self.row(track=track, status=label)
+                self.assertEqual(normalize(raw, 'iclr', year, self.provenance()),
+                                 (None, 'separate_workshop'))
+                row, reason = normalize(raw, 'iclr', year, self.provenance(), True)
+                self.assertIsNone(reason)
+                self.assertEqual((row['venue'], row['year'], row['track'], row['status']),
+                                 ('ICLR', year, track, status))
+                self.assertEqual(row['raw_decision'], label)
+                self.assertEqual(row['metadata']['workshop_venueid'],
+                                 f'ICLR.cc/{year}/Workshop/{group}')
+                self.assertFalse(row['metadata']['original_decision_signature_verified'])
+                self.assertEqual(row['evidence_url'], 'https://example.org/pinned.jsonl#L7')
+                self.assertNotIn('private review', json.dumps(row))
+
+    def test_workshop_label_scope_conflicts_never_become_main_events(self):
+        for year, track, label, _, _ in self.workshop_cases():
+            for enabled in (False, True):
+                for wrong_track in ('main', 'workshop_other'):
+                    self.assertEqual(normalize(self.row(track=wrong_track, status=label),
+                                               'iclr', year, self.provenance(), enabled),
+                                     (None, 'conflicting_workshop_track'))
+                for family, wrong_year in [('icml', year), ('iclr', year + 1)]:
+                    with self.assertRaisesRegex(ValueError, 'conference year'):
+                        normalize(self.row(track=track, status=label), family,
+                                  wrong_year, self.provenance(), enabled)
+        row, reason = normalize(self.row(track='workshop_unknown',
+                                        status='ICLR 2026 Workshop Unknown Reject'),
+                                'iclr', 2026, self.provenance(), True)
+        self.assertIsNone(row)
+        self.assertEqual(reason, 'unmapped_negative_label')
+
+    def test_workshop_collection_and_import_keep_five_tracks_separate_from_main(self):
+        files = {}
+        for index, (year, track, label, _, _) in enumerate(self.workshop_cases()):
+            files.setdefault(f'venues/iclr/iclr{year}.jsonl', []).append(
+                self.row(id=f'Workshop{index:02}', track=track, status=label))
+        files['venues/iclr/iclr2024.jsonl'].append(self.row())
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            default = self.collect(root / 'default', files, families=['iclr'])
+            self.assertEqual(default['normalized_records'], 1)
+            held = [json.loads(line) for line in Path(default['held_file']).read_bytes().split(b'\n') if line]
+            self.assertEqual(len(held), 5)
+            self.assertTrue(all(row['reason'] == 'separate_workshop' for row in held))
+            selected = self.collect(root / 'selected', files, families=['iclr'], include_workshops=True)
+            self.assertEqual(selected['normalized_records'], 6)
+            self.assertEqual(len(selected['files']), 6)
+            self.assertEqual(Path(selected['held_file']).read_bytes(), b'')
+            db = root / 'db.sqlite'
+            for manifest in selected['files']:
+                self.assertEqual(manifest['parser_version'], 3)
+                ingest_file(db, manifest['file'])
+            store = Store(db)
+            try:
+                scopes = set(store.conn.execute('SELECT e.year,e.track,d.status FROM submissions s '
+                                               'JOIN editions e ON e.id=s.edition_id '
+                                               'JOIN decisions d ON d.submission_id=s.id'))
+                expected = {(year, track, status) for year, track, _, status, _ in self.workshop_cases()}
+                self.assertEqual(scopes, expected | {(2024, 'main', 'rejected')})
+                self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(), [])
+            finally:
+                store.conn.close()
+
     def test_mirror_reimport_preserves_one_attempt_and_later_acceptance_elsewhere(self):
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -152,6 +269,35 @@ class PublicOpenReviewTests(unittest.TestCase):
             finally:
                 store.conn.close()
 
+    def test_unicode_separators_in_abstract_survive_audit_and_import(self):
+        for separator in ('\u0085', '\u2028', '\u2029'):
+            with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                abstract = 'First paragraph' + separator + 'Second paragraph'
+                result = self.collect(root / 'source', {
+                    'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]})
+                path = Path(result['files'][0]['file'])
+                data = path.read_bytes()
+                self.assertIn(separator.encode(), data)
+                self.assertEqual(data.count(b'\n'), 1)
+                db = root / 'db.sqlite'
+                store = Store(db)
+                store.conn.close()
+                inventory = root / 'source/public-openreview-inventory.json'
+                before = audit(db, inventory, root / 'before.json')
+                self.assertEqual(before['status'], 'ready_for_serial_import')
+                self.assertEqual(before['records_checked'], 1)
+                ingest_file(db, path)
+                after = audit(db, inventory, root / 'after.json')
+                self.assertEqual(after['status'], 'ready_for_serial_import')
+                self.assertEqual(after['existing_same_edition_attempts'], 1)
+                store = Store(db)
+                try:
+                    self.assertEqual(store.conn.execute(
+                        'SELECT abstract FROM source_records').fetchone()[0], abstract)
+                finally:
+                    store.conn.close()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_research_runtime.py

@@ -0,0 +1,172 @@
+"""Interactions between event identity and previously independent write safeguards."""
+from contextlib import contextmanager, redirect_stdout
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+import uuid
+
+from paper_atlas import querying
+from paper_atlas.audits.coverage import report as coverage_report
+from paper_atlas.collectors.dblp import XML_SHA
+from paper_atlas.curation.annotations import initialize, assert_annotation, retract_annotation, observations
+from paper_atlas.db.backend import open_store
+from paper_atlas.ingestion.event_identity import plan, upgrade
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.postgres.client import connect
+from paper_atlas.postgres.migration import migrate
+from paper_atlas.postgres.schema import quote
+from tests.test_event_identity import legacy, add
+
+
+class RuntimeCases:
+    @contextmanager
+    def store(self):
+        s = open_store(**self.target)
+        try:
+            yield s
+        finally:
+            s.conn.close()
+
+    def query(self, *args):
+        target = (['--db', str(self.target['db'])] if 'db' in self.target else
+                  ['--pg-service', self.target['pg_service'], '--pg-schema', self.target['pg_schema']])
+        with redirect_stdout(io.StringIO()) as out:
+            querying.main([*target, *map(str,args)])
+        return json.loads(out.getvalue())
+
+    def assertion(self, c, paper, request, **changes):
+        args = dict(paper=paper, kind='research_type', namespace='custom:method', value='empirical',
+                    request_id=request, evidence_key='review:'+request, actor='reviewer', reason='Reviewed metadata')
+        args.update(changes)
+        return assert_annotation(c, **args)
+
+    def test_event_upgrade_preserves_annotation_history_and_combined_queries(self):
+        with self.store() as s:
+            initialize(s.conn)
+            event = self.assertion(s.conn,1,'first')
+            retract_annotation(s.conn,event,request_id='retract',actor='reviewer',reason='Corrected')
+            self.assertion(s.conn,1,'second')
+            original = observations(s.conn,1,history=True)
+            s.conn.commit()
+        proposal = plan(**self.target)
+        self.assertEqual(proposal['before']['counts']['annotation_events'],3)
+        result = upgrade(proposal, **self.target)
+        self.assertEqual(result['after']['counts'],proposal['before']['counts'])
+        with self.store() as s:
+            self.assertEqual(observations(s.conn,1,history=True),original)
+            paper,_,_ = add(s,'spring','www-1995-spring','rejected')
+            self.assertEqual(paper,1)
+            s.conn.commit()
+        row = self.query('--venue','WWW','--event-key','www-1995-spring','--status','rejected',
+                         '--research-type','custom:method=empirical','--paper',1,'--annotation-history')[0]
+        self.assertEqual([a['event_key'] for a in row['known_acceptances']],['www-1995-fall'])
+        self.assertEqual({a['latest_status'] for a in row['submissions']},{'accepted','rejected'})
+        self.assertEqual(len(row['annotation_history']),3)
+        self.assertEqual(self.query('--event-key','www-1995-fall','--status','rejected'),[])
+
+    def collected(self, name, event, external, *, incremental=False, unscoped=False):
+        row = dict(source='fixture',external_id=external,url='https://example.org/'+external,
+                   title='Independent contribution '+external,authors=[],abstract='Known abstract',
+                   identifiers={'doi':'10.1234/'+external},venue='WWW',year=1995,track='main',
+                   event_key=event,status='accepted')
+        p = self.root/(name+'.jsonl');p.write_text(json.dumps(row)+'\n')
+        manifest = {k:row[k] for k in ('source','venue','year','track','event_key')}
+        manifest.update(list_url='https://example.org/'+event,count=1,status='partial',
+                        file_sha256=hashlib.sha256(p.read_bytes()).hexdigest())
+        if incremental:
+            manifest.update(coverage_mode='incremental',coverage_previous_count=1,
+                            coverage_previous_expected_count=1,coverage_expected_count=2)
+        if unscoped:
+            manifest.pop('event_key')
+        p.with_suffix('.manifest.json').write_text(json.dumps(manifest))
+        return p
+
+    def ingest(self, path):
+        with redirect_stdout(io.StringIO()):
+            ingest_file(self.target.get('db'),path,**{k:v for k,v in self.target.items() if k != 'db'})
+
+    def test_coverage_labels_both_same_year_events_and_keeps_legacy_readable(self):
+        registry=self.root/'registry.json'
+        registry.write_text(json.dumps({'scope_note':'Fixture only','venues':[{'acronym':'WWW','field':'web'}]}))
+        def snapshot():
+            return coverage_report(self.target.get('db'),registry,**{k:v for k,v in self.target.items() if k!='db'})['venues'][0]
+        self.assertEqual([e['event_key']for e in snapshot()['editions']],[''])
+        upgrade(plan(**self.target),**self.target)
+        with self.store() as s:
+            add(s,'spring','www-1995-spring','rejected');s.conn.commit()
+        result=snapshot();events={e['event_key']:e for e in result['editions']}
+        self.assertEqual(set(events),{'www-1995-fall','www-1995-spring'})
+        self.assertEqual((result['imported_submissions'],result['distinct_papers']),(2,1))
+        self.assertEqual(events['www-1995-fall']['latest_observed_status_counts']['accepted'],1)
+        self.assertEqual(events['www-1995-spring']['latest_observed_status_counts']['rejected'],1)
+        self.assertEqual(events['www-1995-fall']['source_record_counts'][0]['source'],'dblp')
+        self.assertEqual(events['www-1995-spring']['source_record_counts'][0]['source'],'fixture')
+
+    def test_incremental_parent_is_scoped_to_exact_event(self):
+        upgrade(plan(**self.target), **self.target)
+        self.ingest(self.collected('spring','www-1995-spring','spring-one'))
+        self.ingest(self.collected('fall','www-1995-fall','fall-one'))
+        self.ingest(self.collected('append','www-1995-fall','fall-two',incremental=True))
+        with self.store() as s:
+            rows = s.conn.execute('''SELECT e.event_key,es.expected_count,es.imported_count FROM edition_sources es
+                JOIN editions e ON e.id=es.edition_id JOIN sources src ON src.id=es.source_id
+                WHERE src.name='fixture' ORDER BY e.event_key''').fetchall()
+            self.assertEqual([tuple(r) for r in rows],[('www-1995-fall',2,2),('www-1995-spring',1,1)])
+        with self.assertRaisesRegex(ValueError,'event_key'):
+            self.ingest(self.collected('unscoped','www-1995-spring','unwanted',incremental=True,unscoped=True))
+        with self.assertRaisesRegex(ValueError,'parent changed'):
+            self.ingest(self.collected('stale','www-1995-fall','stale',incremental=True))
+        with self.store() as s:
+            self.assertEqual(s.conn.execute("SELECT count(*) FROM source_records r JOIN sources src ON src.id=r.source_id WHERE src.name='fixture'").fetchone()[0],3)
+
+    def test_joint_refresh_keeps_identity_annotations_and_event_query(self):
+        upgrade(plan(**self.target), **self.target)
+        metadata = dict(snapshot_sha256=XML_SHA,proceedings_key='conf/ijcai/2022',
+                        bibliographic_year=2022,bibliography={'booktitle':['IJCAI']})
+        with self.store() as s:
+            initialize(s.conn)
+            paper,record = s.record('dblp','conf/ijcai/Joint22','https://example.org/joint',
+                                   'Joint paper',None,[],{'doi':'10.1234/joint'},metadata)
+            eid = s.decision(paper,record,'IJCAI',2022,'main','conf/ijcai/Joint22','https://example.org/joint','accepted','Accept')
+            self.assertion(s.conn,paper,'joint')
+            s.conn.commit()
+        with self.store() as s:
+            again,r = s.record('dblp','conf/ijcai/Joint22','https://example.org/joint',
+                               'Corrected joint title',None,[],{'doi':'10.1234/joint'},metadata)
+            self.assertEqual(again,paper)
+            self.assertEqual(s.decision(again,r,'IJCAI',2022,'main','conf/ijcai/Joint22','https://example.org/joint','accepted','Accept'),eid)
+            self.assertEqual(s.conn.execute("SELECT count(*) FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym='IJCAI' AND e.year=2022").fetchone()[0],0)
+            s.conn.commit()
+        for venue in ['IJCAI','ECAI']:
+            rows = self.query('--venue',venue,'--year',2022,'--event-key','',
+                              '--research-type','custom:method=empirical','--paper',paper)
+            self.assertEqual(len(rows),1)
+            self.assertEqual(len(rows[0]['submissions']),1)
+            self.assertEqual(rows[0]['submissions'][0]['acronym'],'ECAI+IJCAI')
+
+
+class SQLiteRuntimeTests(RuntimeCases, unittest.TestCase):
+    def setUp(self):
+        t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name)
+        db=self.root/'source.sqlite';legacy(db);self.target={'db':db}
+
+
+@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL service')
+class PostgreSQLRuntimeTests(RuntimeCases, unittest.TestCase):
+    def setUp(self):
+        t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name)
+        db=self.root/'source.sqlite';legacy(db)
+        self.service=os.environ['PAPER_ATLAS_TEST_PG_SERVICE'];self.schema='pa_runtime_'+uuid.uuid4().hex[:12]
+        self.addCleanup(self.cleanup)
+        with redirect_stdout(io.StringIO()):
+            migrate(db,self.service,self.root/'migration.json',schema=self.schema,min_free_gib=0)
+        self.target={'pg_service':self.service,'pg_schema':self.schema}
+
+    def cleanup(self):
+        with connect(self.service) as c:
+            for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(self.schema,)):
+                c.execute('DROP SCHEMA '+quote(name)+' CASCADE')

tests/test_rss_proceedings.py

@@ -0,0 +1,186 @@
+"""Frozen publisher examples, conflicting evidence and preserved attempt identity."""
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors import rss
+from paper_atlas.ingestion.records import ingest_file
+
+FIXTURES = Path(__file__).parent/'fixtures/rss2025'
+OBSERVED = '2026-11-09T10:00:00Z'
+
+
+def fixture(name):
+    return (FIXTURES/name).read_text()
+
+
+class RssProceedingsTests(unittest.TestCase):
+    def setUp(self):
+        self.index = rss.parse_index(fixture('index.html'))
+        self.accepted = rss.parse_program(fixture('accepted.html'), 'Accepted Papers', ('ID', 'Session', 'Title', 'Authors'))
+        self.demos = rss.parse_program(fixture('demos.html'), 'Demos', ('ID', 'Location', 'Title', 'Day', 'Time'))
+
+    def paper(self, pid=1, html=None, entry=None):
+        return rss.parse_paper(html or fixture(f'p{pid:03d}.html'), pid, entry or self.index[pid],
+                               self.accepted[pid], self.demos.get(pid), {'paper': {'observed_at': OBSERVED}})
+
+    def test_frozen_official_census_and_all_fixture_hashes(self):
+        receipts = json.loads((FIXTURES/'receipts.json').read_bytes())['receipts']
+        self.assertEqual(len(receipts), 6)
+        for r in receipts:
+            data = (FIXTURES/r['file']).read_bytes()
+            self.assertEqual((len(data), hashlib.sha256(data).hexdigest()), (r['bytes'], r['sha256']))
+        self.assertEqual(len(self.index), 163)
+        self.assertEqual(set(self.index), set(self.accepted))
+        self.assertEqual(len(self.demos), 15)
+        rss.validate_program(self.index, self.accepted, self.demos, fixture('authorinfo.html'))
+        self.assertFalse(self.demos[93]['Title'].startswith('Demonstrating'))
+
+    def test_complete_authors_abstract_and_source_metadata_are_preserved(self):
+        row = self.paper()
+        self.assertEqual(row['authors'], ['Pascal Roth', 'Jonas Frey', 'Cesar Cadena', 'Marco Hutter'])
+        self.assertTrue(row['abstract'].startswith('Ensuring safe navigation'))
+        self.assertTrue(row['abstract'].endswith('https://github.com/leggedrobotics/fdm.'))
+        self.assertNotIn('Download:', row['abstract'])
+        self.assertEqual(row['identifiers'], {'doi': '10.15607/rss.2025.xxi.001'})
+        self.assertEqual(row['metadata']['publication_date'], '2025/06/21')
+        self.assertIn('AUTHOR', row['metadata']['original_bibtex'])
+        self.assertEqual(row['status'], 'accepted')
+        self.assertNotIn('decision_date', row)
+        self.assertEqual(row['metadata']['decision_date_precision'], 'not_supplied')
+
+    def test_demo_evidence_is_independent_of_title_and_session_topic(self):
+        row = self.paper(2)
+        self.assertEqual(row['track'], 'demo')
+        self.assertEqual(row['metadata']['demo_program_entry']['ID'], '2')
+        self.assertEqual(self.paper()['track'], 'proceedings_unspecified')
+        self.assertEqual({t['namespace'] for t in row['tags']}, {'rss_proceedings:session', 'rss_proceedings:program_role'})
+        self.assertFalse(any('research_type' in t['namespace'] for t in row['tags']))
+        html = fixture('p001.html').replace('Learned Perceptive Forward Dynamics Model', 'Demonstrating a Learned Model')
+        entry = {**self.index[1], 'title': self.index[1]['title'].replace('Learned Perceptive Forward Dynamics Model', 'Demonstrating a Learned Model')}
+        self.assertEqual(self.paper(html=html, entry=entry)['track'], 'proceedings_unspecified')
+
+    def test_missing_abstract_is_explicit_and_multiple_paragraphs_are_complete(self):
+        html = fixture('p001.html')
+        beginning = html.index('<p style="text-align: justify;">')
+        end = html.index('</p>', beginning)
+        empty = html[:beginning]+'<p style="text-align: justify;">'+html[end:]
+        row = self.paper(html=empty)
+        self.assertIsNone(row['abstract'])
+        self.assertEqual(row['metadata']['missing_abstract_reason'], 'empty_official_abstract_section')
+        multi = html.replace('</p>  \n<p>  \n<b>Download:', '</p><p>Second abstract paragraph.</p><p><b>Download:', 1)
+        self.assertTrue(self.paper(html=multi)['abstract'].endswith('Second abstract paragraph.'))
+
+    def test_metadata_drift_author_truncation_and_wrong_paper_doi_fail(self):
+        original = fixture('p001.html')
+        for altered in [original.replace('XXI.001}', 'XXI.002}'),
+                        original.replace('content="2025/06/21"', 'content="2026/06/21"'),
+                        original.replace('<meta name="citation_author" content="Marco Hutter" />', ''),
+                        original.replace('<b>Abstract:</b>', '<b>Summary:</b>'),
+                        original.replace('rss21/p001.pdf', 'rss21/p002.pdf'),
+                        original.replace('citation_volume" content="21', 'citation_volume" content="22')]:
+            with self.subTest(altered=altered[:60]), self.assertRaises(ValueError):
+                self.paper(html=altered)
+
+    def test_duplicate_and_unparsed_index_entries_fail(self):
+        for html in [fixture('index.html').replace('p002.html', 'p001.html'),
+                     fixture('index.html')+'<a href="p999.html">Stray paper</a>',
+                     fixture('index.html').replace( rss.HEADING, 'A different volume')]:
+            with self.subTest(html=html[:40]), self.assertRaises(ValueError):
+                rss.parse_index(html)
+
+    def test_program_title_id_category_and_header_changes_require_review(self):
+        for change in ['title', 'id', 'category']:
+            index = {k: dict(v) for k, v in self.index.items()}
+            demos = {k: dict(v) for k, v in self.demos.items()}
+            if change == 'title':
+                demos[2]['Title'] = 'Another paper'
+            elif change == 'id':
+                del index[1]
+            else:
+                del demos[93]
+            with self.subTest(change=change), self.assertRaises(ValueError):
+                rss.validate_program(index, self.accepted, demos, fixture('authorinfo.html'))
+        with self.assertRaises(ValueError):
+            rss.parse_program(fixture('demos.html').replace('>Location<', '>Other<'), 'Demos', ('ID', 'Location', 'Title', 'Day', 'Time'))
+
+    def snapshot(self, root):
+        root.mkdir()
+        data = (FIXTURES/'p001.html').read_bytes()
+        (root/'paper.html').write_bytes(data)
+        receipt = {'file': 'paper.html', 'url': self.index[1]['url'], 'final_url': self.index[1]['url'],
+                   'http_status': 200, 'content_type': 'text/html', 'bytes': len(data),
+                   'sha256': hashlib.sha256(data).hexdigest(), 'at': OBSERVED}
+        (root/'receipts.json').write_text(json.dumps({'receipts': [receipt]}))
+        return receipt
+
+    def test_frozen_receipt_hash_status_redirect_and_path_guards(self):
+        for failure in ['hash', 'status', 'redirect', 'path', 'duplicate', 'symlink']:
+            with self.subTest(failure=failure), tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)/'source'
+                r = self.snapshot(root)
+                receipts = [r]
+                if failure == 'hash': r['sha256'] = '0'*64
+                elif failure == 'status': r['http_status'] = 404
+                elif failure == 'redirect': r['final_url'] = 'https://example.org/p001.html'
+                elif failure == 'path': r['file'] = '../outside.html'
+                elif failure == 'duplicate': receipts.append(dict(r))
+                else:
+                    (root.parent/'outside.html').write_bytes((root/'paper.html').read_bytes())
+                    (root/'paper.html').unlink()
+                    (root/'paper.html').symlink_to(root.parent/'outside.html')
+                (root/'receipts.json').write_text(json.dumps({'receipts': receipts}))
+                with self.assertRaises(ValueError): rss.Pages(snapshots=root).read(self.index[1]['url'])
+
+    def test_unreviewed_edition_and_existing_output_refuse_work(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            with self.assertRaises(ValueError): rss.collect(2026, tmp, snapshots=FIXTURES)
+            (Path(tmp)/'rss-2025.json').write_text('{}')
+            with self.assertRaises(FileExistsError): rss.collect(2025, tmp, snapshots=FIXTURES)
+
+    def test_native_roundtrip_is_idempotent_with_same_doi_and_distinct_tracks(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            original = self.paper()
+            dblp = {**original, 'source': 'dblp', 'external_id': 'conf/rss/example', 'metadata': {}, 'tags': []}
+            variant = {**original, 'external_id': '2025/other-track', 'track': 'workshop'}
+            for n, record in enumerate([dblp, original, variant, original]):
+                path = root/f'{n}.jsonl'
+                path.write_text(json.dumps(record)+'\n')
+                path.with_suffix('.manifest.json').write_text(json.dumps({'source': record['source'], 'list_url': rss.ROOT, 'count': 1}))
+                ingest_file(root/'sample.sqlite', path)
+            with sqlite3.connect(root/'sample.sqlite') as c:
+                self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+                self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0], 3)
+                self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 2)
+                self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0], 2)
+                self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0], 3)
+                self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
+
+    def test_two_real_papers_collection_commits_evidence_only_after_success(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            index = {pid: self.index[pid] for pid in [1, 2]}
+            accepted = {pid: self.accepted[pid] for pid in [1, 2]}
+            demos = {2: self.demos[2]}
+            with (patch.object(rss, 'EXPECTED', 2), patch.object(rss, 'DEMO_IDS', {2}),
+                  patch.object(rss, 'parse_index', return_value=index),
+                  patch.object(rss, 'parse_program', side_effect=[accepted, demos])):
+                result = rss.collect(2025, root/'out', snapshots=FIXTURES)
+            self.assertEqual((result['records'], len(result['files'])), (2, 2))
+            self.assertFalse(result['database_written'])
+            self.assertFalse(result['pdfs_downloaded'])
+            self.assertEqual(len(result['source_evidence']), 6)
+            for item in result['source_evidence']:
+                self.assertEqual(hashlib.sha256((root/'out'/item['file']).read_bytes()).hexdigest(), item['sha256'])
+            for item in result['files']:
+                self.assertEqual(item['missing_abstract_count'], 0)
+                self.assertEqual(hashlib.sha256(Path(item['file']).read_bytes()).hexdigest(), item['file_sha256'])
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_submission_evidence.py

@@ -133,7 +133,7 @@ class SubmissionTests(unittest.TestCase):
         c.execute("INSERT INTO papers(id,title,title_key) VALUES(1,'Paper','paper')")
         c.execute("INSERT INTO source_records VALUES(1,1,'paper',1,'https://example.org/p','Paper','Abstract','[]','{}',CURRENT_TIMESTAMP)")
         c.execute("INSERT INTO venues VALUES(1,'ICLR','ICLR','machine_learning','conference')")
-        c.execute("INSERT INTO editions VALUES(1,1,2026,'main')")
+        c.execute("INSERT INTO editions(id,venue_id,year,track) VALUES(1,1,2026,'main')")
         c.execute("INSERT INTO submissions VALUES(1,1,1,1,'paper',NULL)")
         c.execute("INSERT INTO decisions VALUES(1,1,'accepted','Accept','https://example.org/decision','2026-01-01T00:00:00Z',NULL)")
         c.commit();c.close()

tests/test_vision.py

@@ -37,6 +37,27 @@ class VisionTests(unittest.TestCase):
                 collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
             self.assertFalse(list(Path(tmp).glob('*.jsonl')))
 
+    def test_fragment_variants_share_one_all_papers_resource(self):
+        root,papers,entry=self.cvf_fixture();all_url=root+'?day=all'
+        pages={root:''.join(f'<a href="{all_url}{fragment}">All</a>' for fragment in ('','#','#papers')),
+               all_url:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>'}
+        pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers[:2]})
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
+            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+        self.assertEqual(manifest['count'],2)
+        self.assertEqual(manifest['index_urls'],[all_url])
+        self.assertEqual([call.args[0] for call in fetch.call_args_list].count(all_url),1)
+        self.assertFalse(any('#' in call.args[0] for call in fetch.call_args_list))
+
+    def test_distinct_all_papers_queries_still_fail_before_publishing(self):
+        root,_,_=self.cvf_fixture();urls=[root+'?day=all',root+'?day=all&view=alternate']
+        html=''.join(f'<a href="{url}">All</a>' for url in urls).encode()
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',return_value=html) as fetch:
+            with self.assertRaisesRegex(ValueError,'Ambiguous all-papers index'):
+                collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+            self.assertEqual(fetch.call_count,1)
+            self.assertFalse(list(Path(tmp).glob('*.jsonl')))
+
     def test_missing_abstract_page_keeps_official_index_metadata_and_marks_partial(self):
         root,papers,entry=self.cvf_fixture();error=HTTPError(papers[0],404,'Not Found',{},io.BytesIO())
         def fetch(url,*args,**kwargs):

tests/test_www_archive.py

@@ -0,0 +1,155 @@
+import hashlib
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+from paper_atlas.collectors.www_archive import (ROOT, SPRING, FALL, Sources, collect, decode,
+                                                early_review, spring_rows)
+
+
+class ArchiveTests(unittest.TestCase):
+    def setUp(self):
+        t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);self.cache=self.root/'cache';self.cache.mkdir()
+        self.pages={ROOT+'www1/PrelimProcs.html':'''<h1>Formal presentations</h1><ol><li><em>Keywords:</em> BUSINESS TRACK?<br><em>Title:</em><b> First contribution</b><br><em>Author:</em>Jane Doe &lt;jane@example.org&gt;<br><li><em>Title:</em><b>Unlinked contribution</b><br><em>Author:</em>John Roe<br></ol>''',
+                    FALL:'<h1>Conference Tracks</h1><ol><li><a href="Agents.html">Agents</a><li><a href="Tutor.html">Tutorials</a></ol>',
+                    ROOT+'www2/Proceedings/Agents.html':'<h3>Panel</h3><dl><dt><a href="a.html">Linked talk</a><dd>Jane Doe<dt>Unlinked talk</a><dd>John Roe</dl>',
+                    ROOT+'www2/Proceedings/Tutor.html':'<h1>Tutorials</h1><dl><dt>Training session<dd>Teacher</dl>'}
+        for track in ['papers','posters']:
+            self.pages[SPRING+track+'/author.html']='<b>Doe, Jane</b><ul><li><a href="01/paper.html">Same contribution</a></li></ul><b>Roe, John</b><a href="01/paper.html">Same contribution</a>'
+            self.pages[SPRING+track+'/abstract.html']='''<h1>Abstracts</h1><a href="#one">Same contribution</a><h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe, John Roe<br>Institute</i><p>Actual abstract with <i>inline emphasis</i> and a separator\u2028 preserved.</p><hr><h2>Another session</h2>'''
+        self.pages[SPRING+'papers/session.html']='<a href="01/paper.html">Same contribution</a>'
+
+    def sources(self):
+        rows=[]
+        for i,(url,html) in enumerate(sorted(self.pages.items())):
+            raw=html.encode()if isinstance(html,str)else html;name='source-'+str(i)+'.html';(self.cache/name).write_bytes(raw)
+            rows.append(dict(url=url,status=200,file=name,bytes=len(raw),sha256=hashlib.sha256(raw).hexdigest()))
+        p=self.cache/'manifest.json';p.write_text(json.dumps({'sources':rows}));sha=hashlib.sha256(p.read_bytes()).hexdigest()
+        return Sources(self.cache,sha),sha
+
+    def test_offline_batch_separates_tracks_review_and_importable_rows(self):
+        _,sha=self.sources();out=self.root/'batch';report=collect(self.cache,sha,out)
+        self.assertEqual(report['ready'],2);self.assertEqual(report['held'],5)
+        queue=json.loads((out/'import-queue.json').read_text())['files'];self.assertEqual(len(queue),2)
+        for name in queue:
+            p=Path(name);rows=[json.loads(l)for l in p.read_text().split('\n') if l];self.assertEqual(len(rows),1)
+            row=rows[0];self.assertEqual(row['event_key'],'www-1995-spring');self.assertEqual(row['authors'],['Jane Doe','John Roe'])
+            self.assertIn(row['track'],['main','poster']);self.assertIn('inline emphasis',row['abstract']);self.assertNotIn('Another session',row['abstract'])
+            self.assertIn('\u2028',row['abstract']);manifest=json.loads(p.with_suffix('.manifest.json').read_text());self.assertEqual(manifest['file_sha256'],hashlib.sha256(p.read_bytes()).hexdigest());self.assertEqual(manifest['status'],'partial')
+        held=[json.loads(l)for l in(out/'review-candidates.jsonl').read_text().split('\n') if l]
+        self.assertTrue(all('status'not in r and 'source_evidence'in r for r in held))
+        self.assertFalse(any('review-candidates'in p for p in queue))
+
+    def test_partial_manifest_preserves_full_listing_size_separately_from_ready_count(self):
+        self.pages[SPRING+'posters/abstract.html']+='<h3><a href="02/other.html">Unlinked poster</a></h3><i>Author: Other Person</i><p>Actual abstract</p>'
+        _,sha=self.sources();out=self.root/'batch';collect(self.cache,sha,out)
+        manifest=json.loads((out/'www-1995-spring-poster.manifest.json').read_text())
+        self.assertEqual((manifest['count'],manifest['source_listing_count'],manifest['held_entry_count']),(1,2,1))
+        self.assertNotIn('coverage_previous_count',manifest)  # Live parent must be independently preflighted.
+
+    def test_malformed_href_requires_exact_id_and_two_clean_index_anchors(self):
+        self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('01/paper.html','01/        paper.html')
+        sources,_=self.sources();rows,held,_=spring_rows(sources,'papers');self.assertEqual(held,[])
+        self.assertEqual(rows[0]['url'],SPRING+'papers/01/paper.html');self.assertTrue(rows[0]['metadata']['url_corroborated_by_author_and_session'])
+        self.assertIn('        ',rows[0]['metadata']['abstract_href_raw'])
+
+    def test_main_index_disagreement_fails_without_publishing_batch(self):
+        self.pages[SPRING+'papers/session.html']='<a href="02/other.html">Other work</a>';_,sha=self.sources();out=self.root/'batch'
+        with self.assertRaisesRegex(ValueError,'ID sets disagree'):collect(self.cache,sha,out)
+        self.assertFalse(out.exists())
+
+    def test_same_entry_id_with_conflicting_clean_url_is_rejected(self):
+        self.pages[SPRING+'papers/session.html']='<a href="01/other.html">Same contribution</a>';sources,_=self.sources()
+        with self.assertRaisesRegex(ValueError,'URLs disagree'):spring_rows(sources,'papers')
+
+    def test_poster_without_linked_author_entry_is_held_even_with_abstract(self):
+        self.pages[SPRING+'posters/author.html']='<b>Doe, Jane</b><a href="#">Same contribution</a>';sources,_=self.sources();rows,held,counts=spring_rows(sources,'posters')
+        self.assertEqual(rows,[]);self.assertEqual(counts['abstract_entries'],1);self.assertEqual(held[0]['reason'],'abstract_entry_missing_linked_author_index')
+
+    def test_unlinked_and_panel_entries_never_become_acceptance_or_rejection(self):
+        sources,_=self.sources();held,counts=early_review(sources);self.assertEqual(counts,{'first_conference_fields':2,'fall_categories':2,'fall_program_entries':3})
+        self.assertTrue(any(r['candidate']['title']=='Unlinked talk'for r in held));self.assertTrue(all('status'not in r for r in held))
+        self.assertEqual(held[0]['candidate']['authors_raw'],['Jane Doe'])
+
+    def test_title_disagreement_goes_to_review_with_source_claims(self):
+        self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('Same contribution','Different contribution');sources,_=self.sources();rows,held,_=spring_rows(sources,'papers')
+        self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'abstract_and_author_index_titles_disagree')
+
+    def test_cache_tampering_and_manifest_hash_mismatch_fail_closed(self):
+        _,sha=self.sources()
+        with self.assertRaisesRegex(ValueError,'manifest checksum'):Sources(self.cache,'0'*64)
+        p=next(self.cache.glob('source-*.html'));p.write_bytes(p.read_bytes()+b'x')
+        with self.assertRaisesRegex(ValueError,'checksum or size'):Sources(self.cache,sha)
+
+    def test_cache_cannot_read_paths_outside_snapshot(self):
+        _,sha=self.sources();p=self.cache/'manifest.json';d=json.loads(p.read_text());d['sources'][0]['file']='../outside.html';p.write_text(json.dumps(d))
+        with self.assertRaisesRegex(ValueError,'plain files'):Sources(self.cache,hashlib.sha256(p.read_bytes()).hexdigest())
+
+    def test_keywords_only_is_missing_abstract_not_invented_summary(self):
+        self.pages[SPRING+'papers/abstract.html']='<h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe</i><p>Keywords: networking, hypertext</p>';sources,_=self.sources();rows,held,_=spring_rows(sources,'papers')
+        self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'missing_abstract')
+
+    def test_old_encoding_and_duplicate_abstract_identity(self):
+        self.assertEqual(decode(b'M\xfcller')[0],'Müller')
+        self.pages[SPRING+'papers/abstract.html']*=2;sources,_=self.sources()
+        with self.assertRaisesRegex(ValueError,'Duplicate abstract'):spring_rows(sources,'papers')
+
+    def test_existing_output_is_never_overwritten(self):
+        _,sha=self.sources();out=self.root/'batch';out.mkdir();(out/'keep').write_text('original')
+        with self.assertRaisesRegex(ValueError,'overwrite'):collect(self.cache,sha,out)
+        self.assertEqual((out/'keep').read_text(),'original')
+
+    def review(self,*,authors=None,author_source='canonical_byline',missing='',page=None):
+        url=SPRING+'posters/01/paper.html'
+        self.pages[url]=page or '<h1>Canonical full title</h1><p>Jane Doe, John Roe</p><p>Keywords: web</p><h2>Abstract</h2><p>Body</p>'
+        _,sha=self.sources();raw=self.pages[url].encode()if isinstance(self.pages[url],str)else self.pages[url]
+        row=dict(entry_id='01',individual_url=url,individual_sha256=hashlib.sha256(raw).hexdigest(),
+                 title='Canonical full title',authors=authors or ['Jane Doe','John Roe'],author_source=author_source,
+                 reason='Reviewed exact canonical entry and its explicit authorship; retain original title variants',missing_abstract_reason=missing)
+        p=self.root/'review.json';p.write_text(json.dumps({'format':1,'source_cache_sha256':sha,'records':[row]}));return sha,p,hashlib.sha256(p.read_bytes()).hexdigest()
+
+    def test_reviewed_variant_keeps_primary_title_and_all_old_title_claims(self):
+        sha,p,h=self.review();out=self.root/'reviewed';r=collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
+        row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['title'],'Canonical full title')
+        self.assertEqual(row['metadata']['abstract_heading_title_raw'],'Same contribution');self.assertEqual(row['metadata']['author_index_titles_raw'],['Same contribution'])
+        self.assertEqual(len(row['metadata']['source_evidence']),3);self.assertEqual(r['reviewed_posters_sha256'],h)
+        self.assertEqual(row['external_id'],'www3:posters:01');self.assertEqual(row['status'],'accepted')
+
+    def test_canonical_byline_can_resolve_unlinked_author_index_without_guessing(self):
+        self.pages[SPRING+'posters/author.html']='<b>Doe, Jane</b><a href="#">Broken index link</a>'
+        sha,p,h=self.review();out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
+        row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['authors'],['Jane Doe','John Roe'])
+        self.assertEqual(row['metadata']['author_index_raw'],[]);self.assertEqual(row['metadata']['author_order'],'canonical_byline')
+
+    def test_reviewed_missing_abstract_is_explicit_partial_not_body_summary(self):
+        self.pages[SPRING+'posters/abstract.html']='<h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe</i><p>Keywords: web</p>'
+        sha,p,h=self.review(missing='Neither listing nor canonical page supplies an explicit abstract')
+        out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
+        row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());m=json.loads((out/'www-1995-spring-poster.manifest.json').read_text())
+        self.assertIsNone(row['abstract']);self.assertEqual((m['missing_abstract_count'],m['status']),(1,'partial'))
+        sha,p,h=self.review()
+        with self.assertRaisesRegex(ValueError,'Missing abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
+        self.assertFalse((self.root/'bad').exists())
+
+    def test_reviewed_names_must_be_in_byline_region_not_citations_or_contact_guess(self):
+        sha,p,h=self.review(authors=['Unrelated Person'],page='<h1>Canonical full title</h1><p>Jane Doe</p><h2>Abstract</h2><p>Unrelated Person cited here</p>')
+        with self.assertRaisesRegex(ValueError,'byline region'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
+        sha,p,h=self.review(author_source='author_index',authors=['Contact Person'])
+        with self.assertRaisesRegex(ValueError,'author-index'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
+
+    def test_review_hash_cache_binding_and_canonical_h1_are_all_required(self):
+        sha,p,h=self.review()
+        with self.assertRaisesRegex(ValueError,'checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256='0'*64)
+        d=json.loads(p.read_text());d['source_cache_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
+        with self.assertRaisesRegex(ValueError,'binding'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
+        d['source_cache_sha256']=sha;d['records'][0]['title']='Wrong title';p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
+        with self.assertRaisesRegex(ValueError,'canonical H1'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
+
+    def test_unlisted_review_entry_and_changed_primary_digest_fail_closed(self):
+        sha,p,h=self.review();d=json.loads(p.read_text());d['records'][0]['individual_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
+        with self.assertRaisesRegex(ValueError,'page checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
+        self.pages[SPRING+'posters/abstract.html']=self.pages[SPRING+'posters/abstract.html'].replace('01/paper.html','02/other.html')
+        self.pages[SPRING+'posters/author.html']=self.pages[SPRING+'posters/author.html'].replace('01/paper.html','02/other.html')
+        sha,p,h=self.review()
+        with self.assertRaisesRegex(ValueError,'outside abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)