Preserve Unicode separators in public metadata preflight

Reviewed-on: #16

.gitea/workflows/check.yml

@@ -26,10 +26,35 @@ jobs:
               root = pathlib.Path(tmp)
               with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
                   tar.extractall(root, filter='data')
-              candidates = list(root.glob('*/test_import_arxiv.py'))
+              candidates = list(root.glob('*/pyproject.toml'))
               if len(candidates)!=1:
                   raise SystemExit('Unexpected repository archive layout')
               src = candidates[0].parent
-              subprocess.run(['python3','-B','-m','unittest','discover','-v'],cwd=src,check=True)
-              subprocess.run(['python3','-B','query.py','--help'],cwd=src,check=True)
+              # Build and installation checks run only on the approved remote runner.
+              venv = root / 'venv'
+              subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
+              python = str(venv / 'bin/python')
+              subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True)
+              testenv = os.environ.copy()
+              service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
+              if service_file.is_file():
+                  testenv.update(PGSERVICEFILE=str(service_file),PAPER_ATLAS_TEST_PG_SERVICE='foundry-research')
+              subprocess.run([python,'-B','-m','unittest','discover','-s','tests','-t','.','-v'],cwd=src,env=testenv,check=True)
+              wheels = root / 'wheels'
+              subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
+              wheel, = wheels.glob('paper_atlas-*.whl')
+              subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
+              outside = root / 'outside-checkout'
+              outside.mkdir()
+              subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','backfill','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
+              smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
+              subprocess.run([python,'-I','-B','-c',smoke],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','query','--db','installed.sqlite','--stats'],cwd=outside,check=True)
           PY

.gitignore

@@ -1,4 +1,8 @@
 __pycache__/
+.venv/
+build/
+dist/
+*.egg-info/
 *.pyc
 *.sqlite
 *.sqlite-*

README.md

@@ -1,313 +1,88 @@
-# Paper Atlas · 论文元数据与投稿历史索引
+# Paper Atlas
 
-Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工具项目**。它不是一次性的下载脚本，也不需要为了采集而持续启动 Web 服务。目标是把 arXiv、重要 AI 会议论文集及公开投稿决定整合成可追溯、可查询、可持续更新的数据库，供后续研究趋势、跨会投稿、主题演化等分析使用。
+**长期维护、按需运行的论文元数据与公开投稿历史工具。**
 
-## 项目目标与当前进度
+Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中，用于查询和后续研究分析。生产数据使用 PostgreSQL，支持从既有 SQLite 索引完整迁移；SQLite 保留离线核验和小型样本用途。它按批次运行，不需要常驻 Web 服务；采集标题、摘要、标识、标签及决定证据，不下载论文全文。
 
-目标包括：完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。
+项目仍在建设中。候选范围包含 43 个会议（含 ACM MM、MLSys），尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准，而不是仓库文件数量。
 
-**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序、CVF/ECVA 视觉会议元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的历史补充；近期 ICLR 公开决定快照已解析，实际主库进度以运行日志与统计为准。arXiv 元数据 ZIP 已完整下载（1,878,831,586 字节），主批次读取 3,233,620 行至 EOF；主库有 3,233,527 条 arXiv 来源记录，93 行差额仍需按原始 ID 独立核验。这些不是去重后的全库论文数，最终全库审计尚未完成。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行，已配置 ces-control:host 对应的 CI 工作流，实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。
+## 项目结构
 
-## 数据范围与来源
-
-- arXiv：官方推荐的 Cornell/Kaggle 元数据快照，包含标识、作者、标题、摘要、类别、版本与外部索引。默认导入整个快照，通过类别筛选 AI 论文，避免永久丢弃交叉领域记录。
-- 会议论文集：优先官方来源，计划接入 OpenReview、PMLR、ACL Anthology、NeurIPS、CVF、ECVA、AAAI、IJCAI；DBLP 用于补充会议目录、标识与出版信息。GitHub 聚合名单只作发现线索，不能作为未经核验的决定证据。
-- 投稿历史：只收录公开可访问并有证据的记录；没有录用信息不等于拒稿，不能据此推断“从未中会”。
-- 当前不下载 PDF、LaTeX、图片或补充材料，不提取论文全文，不推断匿名作者身份。
-
-官方入口：
-
-- [arXiv bulk metadata](https://info.arxiv.org/help/bulk_data/index.html)
-- [arXiv metadata snapshot](https://www.kaggle.com/datasets/Cornell-University/arxiv)
-- [OpenReview 数据接口说明](https://docs.openreview.net/how-to-guides/data-retrieval-and-modification/how-to-get-all-notes-for-submissions-reviews-rebuttals-etc)
-- [PMLR](https://proceedings.mlr.press/)
-- [ACL Anthology 权威 XML 与 API](https://aclanthology.org/info/development/)
-- [NeurIPS](https://proceedings.neurips.cc/)
-- [COLM](https://proceedings.colm.cc/)
-- [CVF](https://openaccess.thecvf.com/)
-- [ECVA](https://www.ecva.net/papers.php)
-- [IJCAI](https://www.ijcai.org/all_proceedings/)
-- [AAAI](https://ojs.aaai.org/index.php/AAAI/issue/archive)
-- [DBLP](https://dblp.org/db/conf/)
-
-## 候选会议覆盖目录
-
-| 领域 | 会议系列 | 主要发现入口 |
-| --- | --- | --- |
-| 机器学习、理论 | NeurIPS、ICML、ICLR、COLM、COLT、AISTATS、UAI、CLeaR | OpenReview、PMLR、NeurIPS、COLM 官方论文集 |
-| 自然语言 | ACL、EMNLP、NAACL、EACL、COLING、CoNLL | ACL Anthology |
-| 视觉 | CVPR、ICCV、ECCV | CVF、ECVA |
-| 综合 AI | AAAI、IJCAI、ECAI | 官方论文集、DBLP |
-| 检索与数据 | KDD、SIGIR、WWW、WSDM、RecSys、CIKM、ICDM、SDM、ECML-PKDD | 官方出版方、DBLP，具体年份入口待逐项登记 |
-| 机器人 | ICRA、IROS、RSS、CoRL | 官方论文集、PMLR、DBLP |
-| 规划、知识与智能体 | ICAPS、KR、AAMAS | 官方论文集、DBLP |
-| 语音与音频 | ICASSP、INTERSPEECH、ISMIR | 官方论文集、ISCA、DBLP |
-| 相关专门领域 | MICCAI、FAccT | 官方出版方、DBLP |
-
-这些是领域覆盖候选，不构成统一排名，也不能据此声称已全部入库。逐年目录以真实举办届次和正式出版清单为准：双年会议不补造不存在的年份；主会、Findings、workshop 分开。`edition_sources` 保存实际采集状态，不能用候选名单代替覆盖率报告。
-
-## 数据模型
-
-| 表 | 用途 |
-| --- | --- |
-| `papers` / `papers_fts` | 稳定论文实体、代表性标题摘要、FTS5 全文检索 |
-| `identifiers` | arXiv ID、DOI 等唯一外部标识 |
-| `source_records` / `sources` | 平台记录、原平台标题摘要与索引元数据、来源 URL 和抓取时间 |
-| `venues` / `venue_aliases` / `editions` | 会议或期刊系列、年份与 track，区分主会、Findings 和 workshop，联合会议用别名查询，不重复生成事件 |
-| `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 |
-| `submissions` / `submission_identifiers` / `submission_sources` | 投稿事件、按届次限定的原平台强标识、多来源证据；同稿的不同投稿尝试分别保留 |
-| `decisions` / `decision_sources` | 明确决定、原始决定日期、来源观察日期及证据，不把缺失决定编造成拒稿 |
-| `submission_merges` / `decision_merges` | 投稿事件与重复决定观察的合并审计，保留来源关联并避免重用已合并的编号 |
-| `tags` / `paper_tags` | 平台标签与人工标签，使用命名空间隔离 |
-| `match_candidates` / `paper_merges` | 潜在重复复核状态、确定合并审计 |
-| `ingest_runs` | 导入批次、状态、错误和恢复位置 |
-
-决定状态为 `accepted`、`rejected`、`desk_rejected`、`withdrawn`、`under_review`、`unknown`。一篇论文可以对应多次投稿和多个决定，不能用单一“最终会议”字段覆盖历史。查询中按观察时间选择最近状态；后续分析需要同时检查来源覆盖率。
-
-当前确定性去重使用 arXiv ID、单个 DOI 和 OpenReview forum ID；跨平台同标题还需至少两位作者吻合，或作者吻合且摘要相同，才自动合并。保留各平台源记录，避免合并后丢失出处。规范化标题只建检索索引，尚未实现模糊匹配流程；**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同，自定义标签应使用 `custom:*` 命名空间。
-
-## 运行与查询
-
-基础导入和查询依赖 Python 3 标准库和带 FTS5 的 SQLite；PMLR 采集另需 `requirements.txt` 中锁定的 PyYAML 6.0.3。无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外，不提交数据库、下载包或访问凭据。
-
-服务器暂存目录：`/data/paper-index/`，快照目录：`/data/paper-index/raw/`，运行代码目录：`/data/paper-index/code/`。这些路径不是对外访问的 API。本地工作区只用于编辑代码、文档和 Git；采集、全量导入和运行检查均在用户确认的服务器执行。
-
-下载完成并验证 ZIP 后，在服务器运行：
-
-```sh
-python3 -B import_arxiv.py \
-  --db /data/paper-index/papers.sqlite \
-  --archive /data/paper-index/raw/arxiv-metadata.zip
+```text
+paper_atlas/
+  cli.py                 统一命令入口
+  collectors/            各来源采集与目录盘点
+  ingestion/             导入、身份回填与单写入协调
+  db/                    数据模型、身份解析与 SQL schema
+  postgres/              PostgreSQL 迁移、原生全文索引和只读连接
+  audits/                来源核验和覆盖报告
+  io/                    网络、快照下载、HTML 与 Git 元数据读取
+  resources/             候选会议目录
+  querying.py            只读查询
+tests/                  回归与集成验证
+docs/                   使用、数据源、数据模型和维护说明
+pyproject.toml          包元数据、依赖和命令注册
+.gitea/workflows/       远程 CI 与安装验证
 ```
 
-导入直接读取压缩包，不落盘展开整个 JSON。每 500 条提交一次事务并保存断点。若同一快照的导入异常中断，用相同命令恢复；恢复阶段需要重新顺序读取并跳过已提交行。不得同时启动多个写入进程。待恢复批次的文件路径、大小和修改时间必须一致；下载快照校验信息应保存在 `raw/arxiv-manifest.json`。
+## 安装与入口
 
-全量导入是有实质磁盘和 I/O 成本的操作。先检查 `df -h`，保持足够空间；数据库、FTS 索引和 WAL 都会增长，压缩包大小不是最终数据库占用。索引和数据库生成属于数据处理，不会构建应用镜像。
+以下安装、测试及批次运行在已确认的远程服务器 / `ces-control:host` runner 上执行。已在 Python 3.14 环境验证；项目要求 Python 3.12 及以上。
 
 ```sh
-# 导入状态、记录量与覆盖状态
-python3 -B query.py --db /data/paper-index/papers.sqlite --stats
-
-# 查询标题或摘要，支持 SQLite FTS5 查询语法
-python3 -B query.py --db /data/paper-index/papers.sqlite --query '"language model"' --limit 20
-
-# 按 arXiv 平台类别筛选
-python3 -B query.py --db /data/paper-index/papers.sqlite --tag arxiv:category=cs.AI
-
-# 会议与决定查询（会议数据导入后才有结果）
-python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 2025 --status rejected
+python3 -m venv .venv
+.venv/bin/python -m pip install '.[postgres]'
+source .venv/bin/activate
+paper-atlas --help
 ```
 
-查询连接使用 SQLite 只读模式，结果输出 JSON。`--stats` 中 `papers` 是按当前规则去重后的论文实体数，`records_by_source` 是各平台已入库记录数，`decision_events_by_status` 是决定事件数。采集文件 manifest 的 `count` 是已采集数量，不能等同已入库数量，也不能把各平台记录数相加当成独立论文数。
+从源码目录也可以使用同一命令树：`python3 -B -m paper_atlas --help`。无需执行散落的脚本文件。
 
-全部目标会议的覆盖报告会保留尚无入库记录的项目，避免只展示已经做完的会议：
+## 常用流程
 
 ```sh
-python3 -B coverage_report.py --db /data/paper-index/papers.sqlite \
+# 查看当前入库量；不同平台记录数不等于独立论文数
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --stats
+
+# 查询标题与摘要
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --query '"language model"'
+
+# 查看一篇论文的来源、标签和所有已知投稿历史
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --paper 123
+
+# 列出全部目标会议，包括尚未入库的项目
+paper-atlas coverage --db /data/paper-index/papers.sqlite \
   --out /data/paper-index/collected/venue-coverage-current.json
-```
 
-报告按会议、年份和 track 列出投稿数、独立论文数及来源清单状态，包含联合会议别名。没有数据的目标明确标为 no_imported_records；已有数据也保持“历史待独立核验”，不会因单个文件导入通过就宣称历年覆盖完成。同一论文可以出现于多个会议或年份，各会议论文数也不能直接相加作为全库数量。
-
-用 `--paper 531782` 可查看单篇论文的所有已知来源、平台标签、投稿届次及带证据的决定历史。会议筛选不区分大小写，状态筛选使用该次投稿的最新已知决定。当前没有公共 Web 页面或网络查询服务。
-
-## 官方会议元数据采集
-
-```sh
-# 单卷示例：ICML 2025，读取官方 gh-pages 元数据，不下载论文正文
-python3 -B collect_pmlr.py --cache /data/paper-index/raw/http-cache \
-  --out /data/paper-index/collected --volume 267
-
-# 待主库完成 arXiv 导入、且没有其他写入进程后，合并会议记录
-python3 -B import_records.py --db /data/paper-index/papers.sqlite \
-  /data/paper-index/collected/pmlr-v267.jsonl
-```
-
-采集输出 JSONL 和对应 manifest，只有明确来源标签才产生相应决定：官方论文集记录为 accepted；PeerRead 使用其明确的 accepted 布尔标签，缺失标签保留 unknown。清单数目不一致时整个文件导入回滚，记录失败批次，不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。旧卷可能把 PDF 放入归档；采集器对前 50 卷使用 Git partial clone 和 sparse checkout，仅取 `_posts` 和卷配置，不执行出版方项目代码。较新归档若超出 64 MiB 限额，也转为此方式。已对原本超限的 AISTATS 2009（第 5 卷）验证采集 84 条且工作区无 PDF。旧 YAML 的 `!binary` 文本、空文档尾标记会安全解析；带年份的 AISTATS/COLT 等历史 shortname 会规范化，避免名称别名漏届；缺失 year 时仅从官方会议 start 日期提取年份。`pmlr-volume-status.json` 逐卷保存结果和候选范围归属，`pmlr-last-run.json` 记录本批实际请求范围，局部重试不能证明全历史完成。缓存默认有效 24 小时；同一次批次可复用缓存，后续更新会重新读取动态入口。
-
-PMLR 的同一出版 ID 可能对应多份 Git 元数据，例如 CoRL 2023 的两篇论文分别存在不同日期的副本。只有论文内容与标识完全一致、差异限于元数据日期时才合为一条，并保留源文件路径和日期版本；冲突副本会使该卷采集失败，保留上一份有效文件。manifest 分别记录 metadata_post_count、duplicate_post_count 和论文 count。
-
-会议导入器还会独立检查每条记录的会议、年份、track 与 manifest 是否一致，并从实际记录计算缺失摘要数；旧 manifest 即使声称摘要完整，也不能让缺失摘要的来源被标为 complete。缺失的源摘要与合并后论文实体是否从其他来源补得摘要分别统计。
-
-PeerRead 采集仅选择作者公开发布的 ICLR 2017 投稿元数据，固定到源仓库 commit。只依据明确的布尔 `accepted` 标签形成 accepted/rejected；空标签为 unknown，不使用其他数据分区中的 probably-rejected 推断。不会保存 reviews 或 parsed_pdfs 正文。
-
-```sh
-python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \
+# 采集与导入分开；这里只采集到独立文件
+paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
   --out /data/paper-index/collected
 ```
 
-数据任务以一次性 systemd 服务运行：`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`；`paper-atlas-pmlr-verify` 用于全索引复核，`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程；会议采集先写独立 JSONL，待 arXiv 主批次结束后再合并；也可以使用 `merge_conferences.py` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行，避免 SSH 断开影响恢复；不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成，需检查批次状态、清单与完整性结果。标签按来源刷新使用 `paper_tags_source_evidence` 索引，避免每条论文都扫描整张标签表。运行中的 arXiv 可以通过同一协调脚本的 `--schema-only` 模式在断点更新索引后恢复；此模式要求已有 arXiv unit 正在运行。
+`paper-atlas collect --help` 查看来源；`paper-atlas import --help` 查看导入方式。完整批次步骤见[使用指南](docs/usage.md)。
 
-### 投稿事件身份与旧数据回填
+共享实例名为 `foundry-postgres`，数据位于 `foundry_data` 库的 `research` schema；其他项目可按权限复用同一组表。库名通过 libpq service 配置、schema 通过 search_path 或 `--pg-schema` 指定，不写死在查询中。PostgreSQL 凭据放在服务器权限为 `0600` 的 libpq service 文件中；命令行只使用服务名称。`paper-atlas maintain postgres --help` 提供迁移入口：先完整核验 21 张业务表，再发布目标 schema。SQLite FTS5 内部表由 PostgreSQL GIN 索引替代。切换后 SQLite 为只读备份；`import records`、`import arxiv` 和 `maintain backfill` 使用 `--pg-service foundry-research` 写入主库，整个写入会话只允许一个协调写入者，会议文件失败时整体回滚并保留失败批次记录。
 
-论文实体与投稿事件分开去重。同一篇论文可以在同一年向同一会议投稿多次，不能仅凭 paper_id、会议、标题相同就把投稿历史压成一次。跨来源只有原平台 forum ID、已验证的投稿编号，或正式发表稿的同届同 track 出版 DOI 等强证据才能关联同一事件。程序展示的内部编号仅在经过独立核验并带有 program_submission_number_verified 标记时才能用作投稿编号。
+## 数据如何组织
 
-新版 ICLR manifest 的 parser_version 为 2，明确区别投稿编号、forum 和展示事件编号。旧版文件应先用缓存重新采集；事件证据回填只接受已入库、与 manifest 的会议/年份/track 和数量一致的文件。队列 JSON 包含 files 及对应的 file_sha256，文件变化后必须重新核验，不能继续使用旧哈希。
+- **论文和投稿分开。** 一篇论文可以多次投稿；某次被拒、后来在其他会议录用，两段历史都保留。
+- **决定必须有证据。** 评分、未出现在录用名单、没有公开决定，均不能推断为 Reject。未发现录用也不等于从未录用。
+- **来源和标签保留出处。** 不同平台标签分开，自定义标签使用 `custom:*`；确认合并时保留源记录与审计。
+- **覆盖范围逐届核对。** 主会、Findings、workshop 等分别记录。单个文件检查通过不能证明整个会议历年完整。
+- **数据库只允许单写入。** 采集任务先写独立文件，核验后由协调器合并；原始数据和数据库均放在仓库外。
 
-```sh
-python3 -B merge_conferences.py --db /data/paper-index/papers.sqlite \
-  --archive /data/paper-index/raw/arxiv-metadata.zip \
-  --backfill-queue /data/paper-index/collected/submission-evidence-queue.json
-```
+## 文档
 
-协调器只维护本项目的 arXiv unit，并对维护过程加互斥锁。若 arXiv 正在运行，会保存断点、应用 schema、回填来源身份、导入本次附加文件、检查外键后恢复；若批次已经读至 EOF 并标记 complete，则不再启动它。若导入停下但未完成，会报错要求先调查，不以“unit 不存在”推断成功或重新全量运行。应以一次性 systemd 任务运行该命令，保持单写入。
+| 内容 | 入口 |
+| --- | --- |
+| 安装后怎么采集、导入和查询 | [使用指南](docs/usage.md) |
+| 数据范围、各采集器和来源限制 | [数据源说明](docs/data-sources.md) |
+| 论文、投稿、决定、标签及去重规则 | [数据模型](docs/data-model.md) |
+| 批次恢复、更新和最终核验 | [运行维护](docs/operations.md) |
+| 模块职责、开发和 CI | [开发说明](docs/development.md) |
 
-首次升级会给已有投稿与决定补上来源关联；身份回填不重新导入标题摘要，不更改原抓取时间。确认同一次投稿后，可以合并镜像事件并保留所有决定及来源。当前稿件的原始强标识也会写入 source_records 元数据，后续源 URL 更新会同步保存。
+## 验证与交付
 
-通用 JSONL 导入支持 decision_events 数组及 observed_at、decision_date、submitted_at。没有取得明确决定时，可用 unknown 状态和空数组仅保存投稿，不生成虚假的决定；`--status unknown` 也能查询这些投稿。历史快照使用其实际观察时间，不能因今天导入就覆盖较新来源的状态；同一观察时间内按原始决定日期排序。反复导入相同证据只补来源关联，不重复制造决定事件。全量结束后仍须单独核验重复候选、冲突决定及来源覆盖，迁移检查不能代替最终全库审计。
+CI 在已授权的远程 runner 上运行测试、构建 wheel，并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤，不因代码测试通过而自动完成。
 
-### 固定版本的补充元数据快照
-
-`download_snapshot.py` 流式下载到 `.part`，核对完整 HTTP 响应、预期大小和可信来源发布的校验和后，才发布正式文件及 manifest。已存在的正式文件会先重新核验；错误校验和不会覆盖它。默认保留 8 GiB 空闲空间，同一输出只允许一个下载进程。
-
-DBLP 官方月度快照用于补充出版目录和历史届次，元数据按 CC0 发布。固定版本可复现，不把每日变动下载链接当作同一快照：
-
-```sh
-python3 -B download_snapshot.py \
-  --url https://drops.dagstuhl.de/storage/artifacts/dblp/xml/2026/dblp-2026-11-01.xml.gz \
-  --md5 18ad5db59592406ec74289975c4b44cb \
-  --output /data/paper-index/raw/dblp-2026-11-01.xml.gz
-```
-
-[官方版本与校验值](https://doi.org/10.4230/dblp.xml.2026-11-01)。下载器另外计算 SHA-256，记录来源 URL、大小和核验时间。DBLP 没有提供的摘要不能编造，出版目录也不能作为拒稿证据。
-
-DBLP 目录盘点先核验固定快照与[官方 DTD](https://doi.org/10.4230/dblp.xml.dtd.2023-06-28)，再流式读取 XML，直至 gzip EOF 检查通过：
-
-```sh
-python3 -B download_snapshot.py \
-  --url https://drops.dagstuhl.de/storage/artifacts/dblp/xml/2023/dblp-2023-06-28.dtd \
-  --md5 96e4283e71886dec2173718957293648 \
-  --output /data/paper-index/raw/dblp-2023-06-28.dtd
-python3 -B catalogue_dblp.py \
-  --archive /data/paper-index/raw/dblp-2026-11-01.xml.gz \
-  --dtd /data/paper-index/raw/dblp-2023-06-28.dtd \
-  --out /data/paper-index/collected/dblp
-```
-
-`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本；`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。存在多个出版卷字段的记录另存 `volume_anomalies`，保留全部值供归属核验；统计中仍只计一条，不任取第一个字段或展开成多篇。候选范围包含历史名称与 workshop，尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识，才能生成入库文件；不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF，也不更改论文主库。
-
-XML 只允许校验过的本地 DTD；官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集，不访问 XML 声明的任意网络地址，并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件，需先调查残留的 `.part` 状态，再决定是否重新运行。
-
-公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定；它是第三方保存的 OpenReview 历史副本，不等于最新官方全量数据：
-
-```sh
-python3 -B download_snapshot.py \
-  --url https://github.com/qhjqhj00/iclr-openreview-reviews/releases/download/v1.0/iclr.tar.bz2 \
-  --sha256 350071daf25ebf1aef854ee1d5dc3a81834e1c468917c82a4f78af5548de34fc \
-  --expected-bytes 512940722 \
-  --output /data/paper-index/raw/iclr-openreview-v1.0.tar.bz2
-```
-
-[快照来源与复现脚本](https://github.com/qhjqhj00/iclr-openreview-reviews)。原包包含评审回复；解析只筛选公开投稿的索引、标题、摘要、平台标签及带证据的明确决定，不提取评审正文、作者邮件或论文全文。必须核对原始 invitation、readers、签名及决定字段，不能根据评分或录用名单缺席推断拒稿。
-
-优先处理近期三届，再单独补全 2020–2023；不以 2017 年的 PeerRead 样本代表近期拒稿覆盖：
-
-```sh
-python3 -B collect_openreview_snapshot.py \
-  --archive /data/paper-index/raw/iclr-openreview-v1.0.tar.bz2 \
-  --out /data/paper-index/collected --year 2024 --year 2025 --year 2026
-```
-
-解析器先核验固定 SHA-256，再流式读取 tar，以临时 SQLite 按年份和 forum 关联投稿及决定，兼容回复先于投稿的归档顺序。只接受明确对 everyone 公开的投稿和具有会议主席签名的 Decision；保留原始决定值、note ID、决定日期、invitation 和签名证据。无明确决定的投稿保留 unknown，不产生拒稿推断。无法关联到公开投稿元数据的公开决定另存 unmatched-decisions 文件，供复核，不擅自创建论文。输出清单分别统计投稿状态、决定事件、缺失摘要和排除原因；重复副本去重，内容冲突会中止。导入时若同一编号指向不同 forum，或同一 forum 指向不同投稿编号，会报冲突并回滚，不强制合并。
-
-该版本是 2026-05-08 发布的第三方历史副本，不代表平台当前状态或全部投稿；缺失决定及公开性筛选均需核验。采集成功也不等于主库已更新，后续应通过单写入协调器导入，并核对同届官方程序与公开决定的差异。
-
-入库前按真实 forum 对照同届官方程序，生成逐年状态、名单缺口、历史决定差异和文本版本差异报告，以及带 SHA-256 的现有程序身份回填队列：
-
-```sh
-python3 -B audit_iclr_snapshot.py --root /data/paper-index/collected \
-  --year 2024 --year 2025 --year 2026 \
-  --report /data/paper-index/collected/iclr-recent-audit.json
-```
-
-该检查要求清单和记录的届次、数量、摘要缺失统计一致，拒绝重复 forum、编号冲突及缺失公开决定证据。报告中的名单差异和 unknown 不会转换成拒稿。不同时间来源的决定保留各自证据，强 forum 标识负责关联同一次投稿；报告通过不等于平台全量覆盖或最终全库去重已经完成。生成的 `*-backfill-queue.json` 仅用于已入库程序记录的身份回填，快照文件仍须由协调器单独导入。
-
-下载异常后先检查对应进程或 CI 任务确已终止，再使用同一参数增加 `--resume`。只有断点身份一致且服务器实际返回正确 Range 响应才续传；不支持时保留断点并报错，不自动重下。正式文件与下载清单均放在仓库外，不提交大型来源包。
-
-### COLM 与 NeurIPS 官方摘要
-
-```sh
-python3 -B collect_proceedings.py --source colm \
-  --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
-python3 -B collect_proceedings.py --source neurips \
-  --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
-```
-
-公共元数据请求对连接重置、超时及不完整响应最多尝试三次，保留原有有效缓存；HTTP 403 等访问拒绝会直接记录失败。声明 Content-Length 与实际响应不符时不发布缓存，已有缓存也受字节上限约束。旧页面没有赋值的 HTML 属性按空值处理，避免某一个空 class 属性阻断整届采集。
-
-采集器从官方根目录发现实际发表届次，仅访问索引和 Abstract HTML 页面，不请求 PDF、补充材料或源码。可用重复 `--year` 限定实际存在的届次；不存在的年份会报错，局部重试保留其他届次的已有状态。每个来源串行请求，实际网络响应后等待 1 秒；缓存命中不额外等待。
-
-COLM 当前目录提供 2024–2026 三届 pre-proceedings，按会议年份保存，保留平台实际出版日期和出版状态，不把晚补刊日期当作会议年份。NeurIPS 当前目录提供 1987–2025，并包含 2025 独立卷及 2021 独立 Datasets and Benchmarks 索引。主会、datasets_benchmarks、position、creative_ai 分开出文件，不混为同一个 track。跨平台自标识仅从出版方 paper-actions 区域链接与 citation 元数据提取，不把摘要里的引用链接当作当前论文 ID。manifest 的 `parser_version` 至少为 2 才允许入库；旧文件须利用缓存重新采集，避免错误身份合并。首次真实样本核对：COLM 2024 官方目录 299 个摘要入口；NeurIPS 1987 为 90 个，作者与摘要均已验证解析。样本成功不代表所有页面采集完成。
-
-`colm-index-status.json`、`neurips-index-status.json` 保存逐索引状态；对应 `*-last-run.json` 保存本次实际选择范围。每个 JSONL 的 manifest 记录索引入口数、track 记录数和缺失摘要数。论文数量与官方页面声明不一致会失败，缺失摘要会标记 partial；不会把页脚、姓名政策或其他正文当作摘要。
-
-这两个采集器只写采集文件，合并仍通过单写入协调流程。全量历史覆盖和所有未实现会议仍待最终核验。
-
-### ACL Anthology 权威 XML
-
-```sh
-python3 -B collect_acl.py \
-  --checkout /data/paper-index/raw/metadata-checkouts/acl-anthology \
-  --out /data/paper-index/collected
-```
-
-`metadata_checkout.py` 对官方仓库做只包含 `data/xml` 的 partial clone/sparse checkout，不获取 PDF，不执行仓库 hooks 或构建。每批固定到实际 Git commit。遍历 XML 的 volume venue 字段选择 ACL、EMNLP、NAACL、EACL、COLING、CoNLL；Findings 按所属会议独立成 track，不按文件名把其他同场 workshop 冒充主会。
-
-新旧 Anthology ID 按官方规则构造，并核对源 DOI。保留 C69、D19-50 至 D19-66、W 系列的历史编号例外，排除编号 0 的前言。XML 混合标记通过 itertext 提取；无摘要保留为空并标记 partial。主会、Findings、workshop、demo、tutorial、industry、shared_task 分开。
-
-联合举办会议保存一个会议事件及多个名称别名。例如 LREC-COLING 的论文可用 COLING 或 LREC 查询，仍只有一次该届投稿记录。可用 `--track main` 限定主会，避免把 Findings 或演示算入主会；别名数据在正常单写入合并时应用 schema。
-
-`acl-collection-status.json` 记录每个 XML 及其 volume 的处理结果和排除原因；`acl-last-run.json` 记录源 commit 与本批范围。`--collection P17` 可限定一个确实存在的 collection。已验证真实 ACL 2017 长文 195 条、短文 107 条，二者摘要完整；其他 track 按源实际缺失情况记录，不能凭样本宣称整个历史完成。
-
-### ICLR 官方公开程序元数据
-
-```sh
-python3 -B collect_iclr.py --cache /data/paper-index/raw/http-cache \
-  --out /data/paper-index/collected
-# 入库后按官方主题检索；主题值以实际来源为准
-python3 -B query.py --db /data/paper-index/papers.sqlite \
-  --venue ICLR --tag 'iclr_program:topic=Reinforcement Learning->Deep RL'
-```
-
-从 ICLR 官方年份导航登记届次，读取对应 Papers 页面明确引用的 orals-posters/abstracts JSON。核对声明数量、分页状态及唯一 event ID；只下载公开程序元数据，不访问用户资料、PDF、海报、幻灯片或视频。`--year` 可限制年份；清单中的已采集数与数据库已入库数分别统计。
-
-官方程序中的 event 数不是论文数：同一篇文章可以有 poster 和 oral 两场展示。程序还会给负 sourceid 的 oral 展示生成占位 OpenReview 链接，这些链接不能作为真实论文标识。采集器核对原投稿编号、标题与作者后合并展示，优先使用正编号记录的真实论文链接；原展示编号和类型保留在源元数据中。
-
-同一投稿编号的展示标题可能有小幅修订。只有负编号 oral 与正编号原稿对应、至少两位作者完全一致且标题相似度满足严格阈值时，才保留为同稿的标题变体；该规则仅在一届官方程序内部生效，不用于跨平台模糊合并。
-
-只将 Conference 来源的论文视为 ICLR 主会录用，排除 Blog 和期刊转会展示，并在审计清单保存排除计数。2020/2021 旧数据允许依据官方主会 paper program 形成录用证据；无 forum ID 时保留 program 身份和匹配候选，不伪造标识。历史年度返回 404 会登记 unavailable，未来未发表届次登记 not_yet_published，其他访问或解析失败登记 blocked；这些状态都不能宣称该年已覆盖。
-
-主题、关键词使用 `iclr_program:topic`、`iclr_program:keyword` 命名空间。通用导入器按源记录刷新标签证据，保留同一论文其他源记录及 `custom:*` 标签。每年的 JSONL、manifest、`iclr-program-year-status.json` 和 `iclr-program-last-run.json` 保存实际范围、独立展示数、主会展示数、论文数、缺失摘要与缺失 forum 数。录用程序不能提供拒稿全集，公开投稿及拒稿补充仍需独立来源。
-
-### CVF / ECVA 视觉会议元数据
-
-```sh
-python3 -B collect_vision.py --source cvf \
-  --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
-python3 -B collect_vision.py --source ecva \
-  --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
-```
-
-CVF 从官方根目录发现 CVPR、ICCV 主会与 Findings，分别保存 track；不把 workshop 或 WACV 算入这些会议。优先跟随页面明确提供的 All Papers 链接，避免只收录默认显示的第一天。ECVA 从官方总目录按论文 URL 的会议与年份分组，当前总目录列有 ECCV 2018、2020、2022、2024。不存在于来源的 `--venue`、`--year` 或二者组合会报错，不补造双年会议届次，也不能因此推断旧届次未举办。
-
-只访问论文索引与摘要 HTML/PHP 页面。作者优先取 citation 元数据，旧页面尊重声明编码；无 citation 作者时取索引中的作者行，排除论文集出版信息。DOI 只取该论文条目所属的 DOI 链接或 citation 字段；冲突会失败，不从摘要引用中抽取当前论文 ID。不请求 PDF、补充材料、视频或代码。
-
-每份清单记录索引入口数、论文数、缺失摘要与作者数，缺失内容保留为空并标记 partial。存在未被标题条目解析的论文链接时会失败，防止静默漏项。逐来源的 discovery、index-status、last-run 文件分别保存发现目录、逐届结果和实际请求范围。可重复使用 `--year` 或 `--venue` 限定重试；其他届次状态会保留。官方开放目录之外的早期届次、尚未刊出的新届次和独立出版方历史数据仍需后续补齐，采集器完成不代表全历史覆盖已完成。
-
-## 长期维护与更新
-
-1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发，遵守来源限流、重试与人机验证要求。
-2. 下载先写 `.part`，确认完整 HTTP 响应和 ZIP 目录，再记录 SHA-256 与时间后切换为正式文件。正式导入会读至 EOF，从而验证成员 CRC。
-3. 每次导入保留来源证据和批次，按会议/年份/track 报告覆盖状态。缺失摘要留空，不编造文本；缺失决定保留未知。
-4. 核验行数、唯一标识、外键、全文查询和 SQLite 完整性后，再将批次视为交付。样本通过不能证明全量正确。
-5. 数据库备份使用 SQLite backup API 或一致性快照，不能在写入过程中只复制 `.sqlite` 主文件而遗漏 WAL。按明确保留策略轮换备份，避免无上限积累。
-6. 不为每次数据更新生成新 Docker 镜像；不常驻下载器；不自动创建定时任务。本项目的更新任务应按需或经用户授权调度。
-7. 代码通过工作分支和 PR 交付；构建或 CI 只能使用用户确认的远程 runner，不在本机触发构建。
-
-## 当前访问限制
-
-2026-11-06 从目标服务器测试：Kaggle 元数据 GET 可下载；OpenReview 会议 group 接口可读，但 notes 接口返回 `ChallengeRequiredError`。拒稿记录导入尚受此限制，不能用录用名单替代全部投稿，也不能声称已覆盖全部拒稿。需通过平台支持的访问方式或有合法来源的公开数据导出继续补全。
-
-已核验的采集文件计数：ICLR 2020–2026 主会 15,534 条、COLM 2024–2026 共 1,573 条，摘要均已取得；候选范围内的 PMLR 文件为 31,428 条，已剔除两份重复出版元数据。ACL Anthology 已选文件为 63,297 条，其中 15,147 条源记录缺摘要。这些是采集文件的来源记录数，不能相加当作去重论文数，也不能代替主库实时统计；实际入库量使用 query.py --stats 查看。
-
-代码在 `codex/metadata-index` 工作分支交付，[Draft PR #1](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/1) 尚未合并，等待全量就绪后用户手动合并。各提交分别由远程 CI 验证，结果以对应提交的 Gitea Actions 检查为准。全量导入、其他会议适配器、去重复核和覆盖率核验继续进行，应随实际结果更新此说明。
+开发通过工作分支和 PR 交付，由用户手动合并。

docs/data-model.md

@@ -0,0 +1,20 @@
+# 数据模型与身份解析
+
+
+| 表 | 用途 |
+| --- | --- |
+| `papers` / `papers_fts` | 稳定论文实体、代表性标题摘要、FTS5 全文检索 |
+| `identifiers` | arXiv ID、DOI 等唯一外部标识 |
+| `source_records` / `sources` | 平台记录、原平台标题摘要与索引元数据、来源 URL 和抓取时间 |
+| `venues` / `venue_aliases` / `editions` | 会议或期刊系列、年份与 track，区分主会、Findings 和 workshop，联合会议用别名查询，不重复生成事件 |
+| `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 |
+| `submissions` / `submission_identifiers` / `submission_sources` | 投稿事件、按届次限定的原平台强标识、多来源证据；同稿的不同投稿尝试分别保留 |
+| `decisions` / `decision_sources` | 明确决定、原始决定日期、来源观察日期及证据，不把缺失决定编造成拒稿 |
+| `submission_merges` / `decision_merges` | 投稿事件与重复决定观察的合并审计，保留来源关联并避免重用已合并的编号 |
+| `tags` / `paper_tags` | 平台标签与人工标签，使用命名空间隔离 |
+| `match_candidates` / `paper_merges` | 潜在重复复核状态、确定合并审计 |
+| `ingest_runs` | 导入批次、状态、错误和恢复位置 |
+
+决定状态为 `accepted`、`rejected`、`desk_rejected`、`withdrawn`、`under_review`、`unknown`。一篇论文可以对应多次投稿和多个决定，不能用单一“最终会议”字段覆盖历史。查询中按观察时间选择最近状态；后续分析需要同时检查来源覆盖率。
+
+当前确定性去重使用 arXiv ID、单个 DOI 和 OpenReview forum ID；跨平台同标题还需至少两位作者吻合，或作者吻合且摘要相同，才自动合并。保留各平台源记录，避免合并后丢失出处。规范化标题只建检索索引，尚未实现模糊匹配流程；**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同，自定义标签应使用 `custom:*` 命名空间。

docs/data-sources.md

@@ -0,0 +1,332 @@
+# 数据范围与采集器
+
+## 数据范围与来源
+
+- arXiv：官方推荐的 Cornell/Kaggle 元数据快照，包含标识、作者、标题、摘要、类别、版本与外部索引。默认导入整个快照，通过类别筛选 AI 论文，避免永久丢弃交叉领域记录。
+- 会议论文集：优先官方来源，计划接入 OpenReview、PMLR、ACL Anthology、NeurIPS、CVF、ECVA、AAAI、IJCAI；DBLP 用于补充会议目录、标识与出版信息。GitHub 聚合名单只作发现线索，不能作为未经核验的决定证据。
+- 投稿历史：只收录公开可访问并有证据的记录；没有录用信息不等于拒稿，不能据此推断“从未中会”。
+- 当前不下载 PDF、LaTeX、图片或补充材料，不提取论文全文，不推断匿名作者身份。
+
+官方入口：
+
+- [arXiv bulk metadata](https://info.arxiv.org/help/bulk_data/index.html)
+- [arXiv metadata snapshot](https://www.kaggle.com/datasets/Cornell-University/arxiv)
+- [OpenReview 数据接口说明](https://docs.openreview.net/how-to-guides/data-retrieval-and-modification/how-to-get-all-notes-for-submissions-reviews-rebuttals-etc)
+- [PMLR](https://proceedings.mlr.press/)
+- [ACL Anthology 权威 XML 与 API](https://aclanthology.org/info/development/)
+- [NeurIPS](https://proceedings.neurips.cc/)
+- [COLM](https://proceedings.colm.cc/)
+- [MLSys](https://proceedings.mlsys.org/)
+- [ACM Multimedia 历史目录](https://dblp.org/db/conf/mm/index.html)
+- [CVF](https://openaccess.thecvf.com/)
+- [ECVA](https://www.ecva.net/papers.php)
+- [IJCAI](https://www.ijcai.org/all_proceedings/)
+- [AAAI](https://ojs.aaai.org/index.php/AAAI/issue/archive)
+- [DBLP](https://dblp.org/db/conf/)
+
+## 候选会议覆盖目录
+
+| 领域 | 会议系列 | 主要发现入口 |
+| --- | --- | --- |
+| 机器学习、理论 | NeurIPS、ICML、ICLR、COLM、COLT、AISTATS、UAI、CLeaR | OpenReview、PMLR、NeurIPS、COLM 官方论文集 |
+| 自然语言 | ACL、EMNLP、NAACL、EACL、COLING、CoNLL | ACL Anthology |
+| 视觉 | CVPR、ICCV、ECCV | CVF、ECVA |
+| 多模态与机器学习系统 | ACM MM、MLSys | ACM / DBLP、MLSys 官方论文集 |
+| 综合 AI | AAAI、IJCAI、ECAI | 官方论文集、DBLP |
+| 检索与数据 | KDD、SIGIR、WWW、WSDM、RecSys、CIKM、ICDM、SDM、ECML-PKDD | 官方出版方、DBLP，具体年份入口待逐项登记 |
+| 机器人 | ICRA、IROS、RSS、CoRL | 官方论文集、PMLR、DBLP |
+| 规划、知识与智能体 | ICAPS、KR、AAMAS | 官方论文集、DBLP |
+| 语音与音频 | ICASSP、INTERSPEECH、ISMIR | 官方论文集、ISCA、DBLP |
+| 相关专门领域 | MICCAI、FAccT | 官方出版方、DBLP |
+
+这些是领域覆盖候选，不构成统一排名，也不能据此声称已全部入库。逐年目录以真实举办届次和正式出版清单为准：双年会议不补造不存在的年份；主会、Findings、workshop 分开。`edition_sources` 保存实际采集状态，不能用候选名单代替覆盖率报告。
+
+## ACM MM 与 MLSys
+
+会议注册表包含 `MM`（别名 `ACMMM`、`ACM MM`）和 `MLSys`。MLSys 官方根目录目前列出 2019–2026 年的 8 卷，直接采集每篇标题、摘要、公开作者和引用元数据：
+
+```sh
+paper-atlas collect mlsys --cache /data/paper-index/raw/http-cache \
+  --out /data/paper-index/collected/mlsys
+```
+
+每年清单核对官方声明数量，缺摘要标为 partial；未知现代轨道、空目录或读取失败不能记为成功。`SysML 2019` 归于同一 MLSys 系列并保留别名。2018 年官方说明接受的投稿为 non-archival，论文集从 2019 年开始；不拼造 2018 年归档卷，也不把未出现在论文集中的投稿判为拒稿。参见 [2018 年官方说明](https://mlsys.org/Conferences/2018/)。
+
+MLSys 老式 `/paper/<year>/hash/…` 与新式 `/paper_files/paper/<year>/hash/…` 链接规范为同一个 `publication_mlsys` 标识，也支持 2023 年实际使用的 `-Abstract-mlsys2023.html` 及对应年份标签；标签年份必须与目录年份一致。DBLP 和官方摘要源共用它，避免同一发表记录成为两个论文实体或两次投稿。只从自身出版社链接提取，不从摘要参考文献提取，也不把发表标识用于合并某次拒稿尝试。共享投稿身份入口要求 MLSys 源具有有效的同年出版社身份，缺失或错年的文件不能生成投稿事件；旧版 DBLP 整理文件可利用已保留的唯一自身出版社链接补得此身份，不需要丢弃来源或依靠标题猜测。
+
+ACM MM 使用固定 DBLP 原始快照和已核验的 proceedings crossref 盘点历史，主会 `ACM Multimedia`、历史编号分卷与同系列 workshop 分开。主会卷也包含 Doctoral Symposium、Reproducibility Companion Papers、Tutorials 等章节（参见 [2025 年目录](https://dblp.org/db/conf/mm/mm2025.html)），而原始 XML 不含逐篇章节，所以这类记录暂标 `proceedings_unspecified`，不能整卷当作 regular 主轨道；后续需逐篇目录证据补齐。旧版候选 JSONL 没有 `mm` / `mlsys` 家族，不能用它假装覆盖新增会议；应从已经下载并校验的原始 XML 重新筛选这两个会议，保留完整扫描数量、卷证据与遗漏原因。DBLP 没有摘要，后续须通过 DOI、真实 forum 等独立标识关联官方或公开元数据，不能将书目导入成功表述为摘要已齐全。
+
+公开拒稿镜像的 `acmmm` 家族映射到 `MM`，只接受明确的负面状态，继续保留镜像与原始签名决定的区别。现有固定 OpenReview 镜像只含 ACM MM 2024 的录用记录，没有据此证明其他年份没有公开拒稿；[2025 年官方 CFP](https://acmmm2025.org/call-for-papers/) 仅说明审稿期间不公开，不能外推为会后永不公开。
+
+## 官方会议元数据采集
+
+```sh
+# 单卷示例：ICML 2025，读取官方 gh-pages 元数据，不下载论文正文
+python3 -B -m paper_atlas collect pmlr --cache /data/paper-index/raw/http-cache \
+  --out /data/paper-index/collected --volume 267
+
+# 待主库完成 arXiv 导入、且没有其他写入进程后，合并会议记录
+python3 -B -m paper_atlas import records --db /data/paper-index/papers.sqlite \
+  /data/paper-index/collected/pmlr-v267.jsonl
+```
+
+采集输出 JSONL 和对应 manifest，只有明确来源标签才产生相应决定：官方论文集记录为 accepted；PeerRead 使用其明确的 accepted 布尔标签，缺失标签保留 unknown。清单数目不一致时整个文件导入回滚，记录失败批次，不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。旧卷可能把 PDF 放入归档；采集器对前 50 卷使用 Git partial clone 和 sparse checkout，仅取 `_posts` 和卷配置，不执行出版方项目代码。较新归档若超出 64 MiB 限额，也转为此方式。已对原本超限的 AISTATS 2009（第 5 卷）验证采集 84 条且工作区无 PDF。旧 YAML 的 `!binary` 文本、空文档尾标记会安全解析；带年份的 AISTATS/COLT 等历史 shortname 会规范化，避免名称别名漏届；缺失 year 时仅从官方会议 start 日期提取年份。`pmlr-volume-status.json` 逐卷保存结果和候选范围归属，`pmlr-last-run.json` 记录本批实际请求范围，局部重试不能证明全历史完成。缓存默认有效 24 小时；同一次批次可复用缓存，后续更新会重新读取动态入口。
+
+PMLR 的同一出版 ID 可能对应多份 Git 元数据，例如 CoRL 2023 的两篇论文分别存在不同日期的副本。只有论文内容与标识完全一致、差异限于元数据日期时才合为一条，并保留源文件路径和日期版本；冲突副本会使该卷采集失败，保留上一份有效文件。manifest 分别记录 metadata_post_count、duplicate_post_count 和论文 count。
+
+会议导入器还会独立检查每条记录的会议、年份、track 与 manifest 是否一致，并从实际记录计算缺失摘要数；旧 manifest 即使声称摘要完整，也不能让缺失摘要的来源被标为 complete。缺失的源摘要与合并后论文实体是否从其他来源补得摘要分别统计。
+
+PeerRead 采集仅选择作者公开发布的 ICLR 2017 投稿元数据，固定到源仓库 commit。只依据明确的布尔 `accepted` 标签形成 accepted/rejected；空标签为 unknown，不使用其他数据分区中的 probably-rejected 推断。不会保存 reviews 或 parsed_pdfs 正文。
+
+### 其他会议的公开拒稿
+
+范围是逐会议、逐年份寻找已公开的拒稿投稿，不能只补录已发表名单。ICML、NeurIPS 部分年份由作者选择是否公开拒稿，因此公开子集不能用于推算完整拒稿数或录用率。其他会议没有在某个镜像中出现 Reject，也不代表该会议没有公开拒稿；需要继续核对官方公开政策、原平台与其他有明确标签的公开快照。
+
+`collect public-openreview` 读取 [PaperCopilot 的公开元数据镜像](https://github.com/papercopilot/openreview/tree/41ee277825d685bd78e88e48b69da6aead9a69bb)，固定版本并核验每个文件的 Git blob 哈希。默认盘点该版本截至 2026 年的非 ICLR 文件；ICLR 默认沿用独立的公开原始投稿/决定快照，需要补查时可显式指定 `--family iclr`。只保留明确的 Reject、Desk rejected、Withdrawn 标签，不根据 Submitted、Active、Pending、TBD、分数或录用名单缺席推断拒稿。
+
+```sh
+paper-atlas collect public-openreview --out /data/paper-index/collected/public-openreview-v1
+paper-atlas audit public-openreview --db /data/paper-index/papers.sqlite \
+  --inventory /data/paper-index/collected/public-openreview-v1/public-openreview-inventory.json \
+  --out /data/paper-index/collected/public-openreview-preimport.json
+```
+
+输出按会议、届次及 main、position、datasets_benchmarks 分文件。workshop 默认进入单独的 held 清单；有明确 workshop 名称时可通过 `--include-workshops` 单列采集，不合并到主会。Retraction、未映射的轨道或负面标签也保留在 `held-records.jsonl`，不强改为 Reject 或 Withdrawn。当前决定表没有独立 retracted 状态，因此撤回发表记录仍需后续专门建模。
+
+该来源明确标记为 `curated_public_status_assertion`：保留源 commit、文件哈希、行号、原标签和 forum ID，不声称已核验原始 PC 签名。只保存标题、摘要、公开作者名、关键词、领域、索引与决定来源，不缓存镜像中的个人资料、邮箱、性别、评审内容或 PDF。所有输出清单均为 partial；盘点中的零条记录仅描述这个固定镜像。
+
+入库前审计核验文件哈希、数量、轨道和来源声明，并用真实 forum ID 对比同届同轨道已有投稿。存在不同决定时阻止整批导入，交由来源和时序核查；另一会议的后续录用属于独立投稿历史，不构成该次拒稿的冲突。同一 forum 的镜像不新增投稿事件，但论文实体的跨平台去重仍须在全量结束后核验。审计通过的报告包含带 SHA-256 的导入队列；应在单写入维护锁下及时串行导入，主库变化后重新审计。
+
+ICML 2026 另有[公开原始投稿快照](https://github.com/HemantKumar01/ICML-2026-data/blob/1a1b9d973a8dfbf254f740fb8100a2fcf36757ea/data/rejected.json)，包含 214 条投稿根记录。`collect icml-public-archive` 固定该 commit、文件长度及 Git blob 哈希，逐条要求 `readers` 包含 everyone、forum 等于自身 ID、域及投稿邀请属于主会、平台 `venueid` 明确为 `ICML.cc/2026/Conference/Rejected_Submission`。不能仅根据显示文案 Submitted to ICML 2026 推断拒稿。
+
+```sh
+paper-atlas collect icml-public-archive --out /data/paper-index/collected/icml-2026-public-archive
+paper-atlas audit public-openreview --db /data/paper-index/papers.sqlite \
+  --inventory /data/paper-index/collected/icml-2026-public-archive/public-openreview-inventory.json \
+  --out /data/paper-index/collected/icml-2026-public-preimport.json
+```
+
+这批证据标记为 `public_submission_venueid`，保留逐投稿许可及快照出处；它与整理者的 Reject 标签、原始签名决定分别表述。记录修改时间不当作决定日期，观察时间采用快照上界，不采用今天的导入时间。214 是这个公开 opt-in 子集的投稿数，不能当作会议全部拒稿数，也不能直接当作去重论文实体数。该采集器同样只提取元数据与标题摘要，丢弃 authorids、个人资料、评审和 PDF 字段。
+
+```sh
+python3 -B -m paper_atlas collect peerread --cache /data/paper-index/raw/http-cache \
+  --out /data/paper-index/collected
+```
+
+数据任务以一次性 systemd 服务运行：`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`；`paper-atlas-pmlr-verify` 用于全索引复核，`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程；会议采集先写独立 JSONL，待 arXiv 主批次结束后再合并；也可以使用 `paper-atlas maintain merge` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行，避免 SSH 断开影响恢复；不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成，需检查批次状态、清单与完整性结果。标签按来源刷新使用 `paper_tags_source_evidence` 索引，避免每条论文都扫描整张标签表。运行中的 arXiv 可以通过同一协调脚本的 `--schema-only` 模式在断点更新索引后恢复；此模式要求已有 arXiv unit 正在运行。
+
+### 投稿事件身份与旧数据回填
+
+论文实体与投稿事件分开去重。同一篇论文可以在同一年向同一会议投稿多次，不能仅凭 paper_id、会议、标题相同就把投稿历史压成一次。跨来源只有原平台 forum ID、已验证的投稿编号，或正式发表稿的同届同 track 出版 DOI 等强证据才能关联同一事件。程序展示的内部编号仅在经过独立核验并带有 program_submission_number_verified 标记时才能用作投稿编号。
+
+新版 ICLR manifest 的 parser_version 为 2，明确区别投稿编号、forum 和展示事件编号。旧版文件应先用缓存重新采集；事件证据回填只接受已入库、与 manifest 的会议/年份/track 和数量一致的文件。队列 JSON 包含 files 及对应的 file_sha256，文件变化后必须重新核验，不能继续使用旧哈希。
+
+```sh
+python3 -B -m paper_atlas maintain merge --db /data/paper-index/papers.sqlite \
+  --archive /data/paper-index/raw/arxiv-metadata.zip \
+  --backfill-queue /data/paper-index/collected/submission-evidence-queue.json
+```
+
+协调器只维护本项目的 arXiv unit，并对维护过程加互斥锁。若 arXiv 正在运行，会保存断点、应用 schema、回填来源身份、导入本次附加文件、检查外键后恢复；若批次已经读至 EOF 并标记 complete，则不再启动它。若导入停下但未完成，会报错要求先调查，不以“unit 不存在”推断成功或重新全量运行。应以一次性 systemd 任务运行该命令，保持单写入。
+
+首次升级会给已有投稿与决定补上来源关联；身份回填不重新导入标题摘要，不更改原抓取时间。确认同一次投稿后，可以合并镜像事件并保留所有决定及来源。当前稿件的原始强标识也会写入 source_records 元数据，后续源 URL 更新会同步保存。
+
+通用 JSONL 导入支持 decision_events 数组及 observed_at、decision_date、submitted_at。没有取得明确决定时，可用 unknown 状态和空数组仅保存投稿，不生成虚假的决定；`--status unknown` 也能查询这些投稿。历史快照使用其实际观察时间，不能因今天导入就覆盖较新来源的状态；同一观察时间内按原始决定日期排序。反复导入相同证据只补来源关联，不重复制造决定事件。全量结束后仍须单独核验重复候选、冲突决定及来源覆盖，迁移检查不能代替最终全库审计。
+
+### 固定版本的补充元数据快照
+
+`paper-atlas download` 流式下载到 `.part`，核对完整 HTTP 响应、预期大小和可信来源发布的校验和后，才发布正式文件及 manifest。已存在的正式文件会先重新核验；错误校验和不会覆盖它。默认保留 8 GiB 空闲空间，同一输出只允许一个下载进程。
+
+DBLP 官方月度快照用于补充出版目录和历史届次，元数据按 CC0 发布。固定版本可复现，不把每日变动下载链接当作同一快照：
+
+```sh
+python3 -B -m paper_atlas download \
+  --url https://drops.dagstuhl.de/storage/artifacts/dblp/xml/2026/dblp-2026-11-01.xml.gz \
+  --md5 18ad5db59592406ec74289975c4b44cb \
+  --output /data/paper-index/raw/dblp-2026-11-01.xml.gz
+```
+
+[官方版本与校验值](https://doi.org/10.4230/dblp.xml.2026-11-01)。下载器另外计算 SHA-256，记录来源 URL、大小和核验时间。DBLP 没有提供的摘要不能编造，出版目录也不能作为拒稿证据。
+
+DBLP 目录盘点先核验固定快照与[官方 DTD](https://doi.org/10.4230/dblp.xml.dtd.2023-06-28)，再流式读取 XML，直至 gzip EOF 检查通过：
+
+```sh
+python3 -B -m paper_atlas download \
+  --url https://drops.dagstuhl.de/storage/artifacts/dblp/xml/2023/dblp-2023-06-28.dtd \
+  --md5 96e4283e71886dec2173718957293648 \
+  --output /data/paper-index/raw/dblp-2023-06-28.dtd
+python3 -B -m paper_atlas catalogue dblp \
+  --archive /data/paper-index/raw/dblp-2026-11-01.xml.gz \
+  --dtd /data/paper-index/raw/dblp-2023-06-28.dtd \
+  --out /data/paper-index/collected/dblp
+```
+
+`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本；`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。存在多个出版卷字段的记录另存 `volume_anomalies`，保留全部值供归属核验；统计中仍只计一条，不任取第一个字段或展开成多篇。候选范围包含历史名称与 workshop，尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识，才能生成入库文件；不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF，也不更改论文主库。
+
+目录盘点完成后，可按已核对的书目名称整理指定会议的分届文件。例如先补齐尚无主库记录的会议：
+
+```sh
+paper-atlas collect dblp \
+  --catalogue /data/paper-index/collected/dblp/dblp-catalogue.sqlite \
+  --archive /data/paper-index/raw/dblp-2026-11-01.xml.gz \
+  --dtd /data/paper-index/raw/dblp-2023-06-28.dtd \
+  --venue AAAI --venue IJCAI --venue KDD \
+  --out /data/paper-index/collected/dblp-normalized
+```
+
+也可用 `--candidates` 指定已通过清单 SHA-256 核验的候选文件；完整 XML 模式则可包含初次候选族之外的实际目录，例如 CLeaR 的 `clear2`。规则按书目名、crossref 论文集和会议年核对：不把另一个 `CLEAR` 或 Adaptive Agents 会议误标为 CLeaR/AAMAS；联合会议保留名称别名，出版年与会议年分别保存。workshop 默认单独留待审核，可显式使用 `--include-workshops`；其他已识别的官方附属 track 单列。
+
+输出按会议/年份/track 分文件，附 manifest、`dblp-normalization.json` 范围审查统计和带哈希的 `dblp-import-queue.json`。不能识别的书目、年份或归属会逐类登记，不默认算作主会。DBLP 不含摘要，因此输出摘要为空、清单为 partial；需要通过有证据的跨来源关联补齐论文实体摘要。书目发表证据与原始审稿决定分别保留，不依据书目缺席推断拒稿。已存在其他出版来源的届次，还须先回填其权威身份并核验镜像记录，不能直接累加投稿数。
+
+部分早期 SIGIR、CIKM、IJCAI 出版卷使用两位年份键；只有论文集标题明确给出唯一完整年份且与短键相符时才采用，不推测世纪。WWW 特殊兴趣、Alternate track 单列；同卷混合 Applied Data Science、Demo、Industrial 且不能逐篇区分时使用 `proceedings_unspecified`，不把整卷归为某一个具体 track。
+
+书目中的前后页、目录组合和明确会务标题不产生论文录用记录；缺作者的赞助、委员会、活动汇总等标题单独排除。WWW 2019 companion 的匿名合成活动条目归入待核查材料，原始快照仍保留。范围报告分别统计这些原因；单纯缺作者并不排除真实研究论文，也不补造作者。
+
+XML 只允许校验过的本地 DTD；官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集，不访问 XML 声明的任意网络地址，并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件，需先调查残留的 `.part` 状态，再决定是否重新运行。
+
+公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定；它是第三方保存的 OpenReview 历史副本，不等于最新官方全量数据：
+
+```sh
+python3 -B -m paper_atlas download \
+  --url https://github.com/qhjqhj00/iclr-openreview-reviews/releases/download/v1.0/iclr.tar.bz2 \
+  --sha256 350071daf25ebf1aef854ee1d5dc3a81834e1c468917c82a4f78af5548de34fc \
+  --expected-bytes 512940722 \
+  --output /data/paper-index/raw/iclr-openreview-v1.0.tar.bz2
+```
+
+[快照来源与复现脚本](https://github.com/qhjqhj00/iclr-openreview-reviews)。原包包含评审回复；解析只筛选公开投稿的索引、标题、摘要、平台标签及带证据的明确决定，不提取评审正文、作者邮件或论文全文。必须核对原始 invitation、readers、签名及决定字段，不能根据评分或录用名单缺席推断拒稿。
+
+优先处理近期三届，再单独补全 2020–2023；不以 2017 年的 PeerRead 样本代表近期拒稿覆盖：
+
+```sh
+python3 -B -m paper_atlas collect iclr-snapshot \
+  --archive /data/paper-index/raw/iclr-openreview-v1.0.tar.bz2 \
+  --out /data/paper-index/collected --year 2024 --year 2025 --year 2026
+```
+
+解析器先核验固定 SHA-256，再流式读取 tar，以临时 SQLite 按年份和 forum 关联投稿及决定，兼容回复先于投稿的归档顺序。只接受明确对 everyone 公开的投稿和具有会议主席签名的 Decision；保留原始决定值、note ID、决定日期、invitation 和签名证据。无明确决定的投稿保留 unknown，不产生拒稿推断。无法关联到公开投稿元数据的公开决定另存 unmatched-decisions 文件，供复核，不擅自创建论文。输出清单分别统计投稿状态、决定事件、缺失摘要和排除原因；重复副本去重，内容冲突会中止。导入时若同一编号指向不同 forum，或同一 forum 指向不同投稿编号，会报冲突并回滚，不强制合并。
+
+该版本是 2026-05-08 发布的第三方历史副本，不代表平台当前状态或全部投稿；缺失决定及公开性筛选均需核验。采集成功也不等于主库已更新，后续应通过单写入协调器导入，并核对同届官方程序与公开决定的差异。
+
+入库前按真实 forum 对照同届官方程序，生成逐年状态、名单缺口、历史决定差异和文本版本差异报告，以及带 SHA-256 的现有程序身份回填队列：
+
+```sh
+python3 -B -m paper_atlas audit iclr --root /data/paper-index/collected \
+  --year 2024 --year 2025 --year 2026 \
+  --report /data/paper-index/collected/iclr-recent-audit.json
+```
+
+该检查要求清单和记录的届次、数量、摘要缺失统计一致，拒绝重复 forum、编号冲突及缺失公开决定证据。报告中的名单差异和 unknown 不会转换成拒稿。不同时间来源的决定保留各自证据，强 forum 标识负责关联同一次投稿；报告通过不等于平台全量覆盖或最终全库去重已经完成。生成的 `*-backfill-queue.json` 仅用于已入库程序记录的身份回填，快照文件仍须由协调器单独导入。
+
+下载异常后先检查对应进程或 CI 任务确已终止，再使用同一参数增加 `--resume`。只有断点身份一致且服务器实际返回正确 Range 响应才续传；不支持时保留断点并报错，不自动重下。正式文件与下载清单均放在仓库外，不提交大型来源包。
+
+### COLM 与 NeurIPS 官方摘要
+
+```sh
+python3 -B -m paper_atlas collect colm \
+  --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+python3 -B -m paper_atlas collect neurips \
+  --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+```
+
+公共元数据请求对连接重置、超时及不完整响应最多尝试三次，保留原有有效缓存；HTTP 403 等访问拒绝会直接记录失败。声明 Content-Length 与实际响应不符时不发布缓存，已有缓存也受字节上限约束。旧页面没有赋值的 HTML 属性按空值处理，避免某一个空 class 属性阻断整届采集。
+
+采集器从官方根目录发现实际发表届次，仅访问索引和 Abstract HTML 页面，不请求 PDF、补充材料或源码。可用重复 `--year` 限定实际存在的届次；不存在的年份会报错，局部重试保留其他届次的已有状态。每个来源串行请求，实际网络响应后等待 1 秒；缓存命中不额外等待。
+
+COLM 当前目录提供 2024–2026 三届 pre-proceedings，按会议年份保存，保留平台实际出版日期和出版状态，不把晚补刊日期当作会议年份。NeurIPS 当前目录提供 1987–2025，并包含 2025 独立卷及 2021 独立 Datasets and Benchmarks 索引。主会、datasets_benchmarks、position、creative_ai 分开出文件，不混为同一个 track。跨平台自标识仅从出版方 paper-actions 区域链接与 citation 元数据提取，不把摘要里的引用链接当作当前论文 ID。manifest 的 `parser_version` 至少为 2 才允许入库；旧文件须利用缓存重新采集，避免错误身份合并。首次真实样本核对：COLM 2024 官方目录 299 个摘要入口；NeurIPS 1987 为 90 个，作者与摘要均已验证解析。样本成功不代表所有页面采集完成。
+
+`colm-index-status.json`、`neurips-index-status.json` 保存逐索引状态；对应 `*-last-run.json` 保存本次实际选择范围。每个 JSONL 的 manifest 记录索引入口数、track 记录数和缺失摘要数。论文数量与官方页面声明不一致会失败，缺失摘要会标记 partial；不会把页脚、姓名政策或其他正文当作摘要。
+
+这两个采集器只写采集文件，合并仍通过单写入协调流程。全量历史覆盖和所有未实现会议仍待最终核验。
+
+### ACL Anthology 权威 XML
+
+```sh
+python3 -B -m paper_atlas collect acl \
+  --checkout /data/paper-index/raw/metadata-checkouts/acl-anthology \
+  --out /data/paper-index/collected
+```
+
+`paper_atlas/io/git_metadata.py` 对官方仓库做只包含 `data/xml` 的 partial clone/sparse checkout，不获取 PDF，不执行仓库 hooks 或构建。每批固定到实际 Git commit。遍历 XML 的 volume venue 字段选择 ACL、EMNLP、NAACL、EACL、COLING、CoNLL；Findings 按所属会议独立成 track，不按文件名把其他同场 workshop 冒充主会。
+
+新旧 Anthology ID 按官方规则构造，并核对源 DOI。保留 C69、D19-50 至 D19-66、W 系列的历史编号例外，排除编号 0 的前言。XML 混合标记通过 itertext 提取；无摘要保留为空并标记 partial。主会、Findings、workshop、demo、tutorial、industry、shared_task 分开。
+
+联合举办会议保存一个会议事件及多个名称别名。例如 LREC-COLING 的论文可用 COLING 或 LREC 查询，仍只有一次该届投稿记录。可用 `--track main` 限定主会，避免把 Findings 或演示算入主会；别名数据在正常单写入合并时应用 schema。
+
+`acl-collection-status.json` 记录每个 XML 及其 volume 的处理结果和排除原因；`acl-last-run.json` 记录源 commit 与本批范围。`--collection P17` 可限定一个确实存在的 collection。已验证真实 ACL 2017 长文 195 条、短文 107 条，二者摘要完整；其他 track 按源实际缺失情况记录，不能凭样本宣称整个历史完成。
+
+### ICLR 官方公开程序元数据
+
+```sh
+python3 -B -m paper_atlas collect iclr-program --cache /data/paper-index/raw/http-cache \
+  --out /data/paper-index/collected
+# 入库后按官方主题检索；主题值以实际来源为准
+python3 -B -m paper_atlas query --db /data/paper-index/papers.sqlite \
+  --venue ICLR --tag 'iclr_program:topic=Reinforcement Learning->Deep RL'
+```
+
+从 ICLR 官方年份导航登记届次，读取对应 Papers 页面明确引用的 orals-posters/abstracts JSON。核对声明数量、分页状态及唯一 event ID；只下载公开程序元数据，不访问用户资料、PDF、海报、幻灯片或视频。`--year` 可限制年份；清单中的已采集数与数据库已入库数分别统计。
+
+官方程序中的 event 数不是论文数：同一篇文章可以有 poster 和 oral 两场展示。程序还会给负 sourceid 的 oral 展示生成占位 OpenReview 链接，这些链接不能作为真实论文标识。采集器核对原投稿编号、标题与作者后合并展示，优先使用正编号记录的真实论文链接；原展示编号和类型保留在源元数据中。
+
+同一投稿编号的展示标题可能有小幅修订。只有负编号 oral 与正编号原稿对应、至少两位作者完全一致且标题相似度满足严格阈值时，才保留为同稿的标题变体；该规则仅在一届官方程序内部生效，不用于跨平台模糊合并。
+
+只将 Conference 来源的论文视为 ICLR 主会录用，排除 Blog 和期刊转会展示，并在审计清单保存排除计数。2020/2021 旧数据允许依据官方主会 paper program 形成录用证据；无 forum ID 时保留 program 身份和匹配候选，不伪造标识。历史年度返回 404 会登记 unavailable，未来未发表届次登记 not_yet_published，其他访问或解析失败登记 blocked；这些状态都不能宣称该年已覆盖。
+
+ICLR 2020 的现代程序 JSON 没有真实 forum 链接，数字 event ID 也不能拼接为旧版海报地址。该年未核验身份的记录先使用官方 Papers 索引地址。旧版官方页面实际引用的 [papers.json](https://iclr.cc/virtual_2020/papers.json) 提供 687 个 forum；可用固定大小及 SHA-256 校验的副本进行独立身份核对：
+
+```sh
+paper-atlas download --url https://iclr.cc/virtual_2020/papers.json \
+  --output /data/paper-index/raw/iclr-2020-legacy-papers.json \
+  --expected-bytes 391687 \
+  --sha256 2405f2da6e2641fa600762f7d7cdf7c9d27f4d0ae65799168f3aee46036d30d2
+
+paper-atlas audit iclr-legacy \
+  --program /data/paper-index/collected/iclr-program-2020-main.jsonl \
+  --snapshot /data/paper-index/collected/openreview-snapshot-2020-main.jsonl \
+  --legacy /data/paper-index/raw/iclr-2020-legacy-papers.json \
+  --out /data/paper-index/collected/iclr-2020-verified
+```
+
+核对同时要求同届公开录用、唯一投稿编号和官方 forum 清单一致；编号只是候选关联，还必须有独立的标题/摘要/公开作者证据。旧版作者缩写或姓名格式差异不会放宽全库作者匹配规则。一篇 Explain Your Move 有三个来源中的三种标题，使用逐个核对的精确标题组合及至少两名共同公开作者作例外，不启用通用模糊标题合并。
+
+输出保留原 `2020/program-编号` 来源 ID、标题、摘要和作者，补充真实 forum、核验证据和旧版官方海报 URL。它不写数据库，也不修改输入文件。先在维护锁下使用输出的 `iclr-2020-backfill-queue.json` 给已有程序投稿回填身份，再导入核验后的程序文件与原始投稿快照；这样补充拒稿不会把已有录用事件再建一次。`iclr-2020-snapshot-queue.json` 和审计报告均记录待用文件的哈希，合并前仍须复核。
+
+主题、关键词使用 `iclr_program:topic`、`iclr_program:keyword` 命名空间。通用导入器按源记录刷新标签证据，保留同一论文其他源记录及 `custom:*` 标签。每年的 JSONL、manifest、`iclr-program-year-status.json` 和 `iclr-program-last-run.json` 保存实际范围、独立展示数、主会展示数、论文数、缺失摘要与缺失 forum 数。录用程序不能提供拒稿全集，公开投稿及拒稿补充仍需独立来源。
+
+### CVF / ECVA 视觉会议元数据
+
+```sh
+python3 -B -m paper_atlas collect cvf \
+  --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+python3 -B -m paper_atlas collect ecva \
+  --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+```
+
+CVF 从官方根目录发现 CVPR、ICCV 主会与 Findings，分别保存 track；不把 workshop 或 WACV 算入这些会议。优先跟随页面明确提供的 All Papers 链接；只有逐日目录时遍历全部明确链接的日期，再按论文 URL 合并重复条目并检查标题、作者、DOI 一致性。`.py` 与无后缀的同届入口可关联，不跟随其他会议、年份或站点的日期链接。ECVA 从官方总目录按论文 URL 的会议与年份分组，当前总目录列有 ECCV 2018、2020、2022、2024。不存在于来源的 `--venue`、`--year` 或二者组合会报错，不补造双年会议届次，也不能因此推断旧届次未举办。
+
+只访问论文索引与摘要 HTML/PHP 页面。作者优先取 citation 元数据，旧页面尊重声明编码；无 citation 作者时取索引中的作者行，排除论文集出版信息。DOI 只取该论文条目所属的 DOI 链接或 citation 字段；冲突会失败，不从摘要引用中抽取当前论文 ID。不请求 PDF、补充材料、视频或代码。
+
+每份清单记录索引入口数、论文数、缺失摘要与作者数，缺失内容保留为空并标记 partial。存在未被标题条目解析的论文链接时会失败，防止静默漏项。逐来源的 discovery、index-status、last-run 文件分别保存发现目录、逐届结果和实际请求范围。可重复使用 `--year` 或 `--venue` 限定重试；其他届次状态会保留。官方开放目录之外的早期届次、尚未刊出的新届次和独立出版方历史数据仍需后续补齐，采集器完成不代表全历史覆盖已完成。
+
+单个摘要页面 404 时，仍保存官方目录提供的题目、作者和发表证据，摘要留空并记录失败 URL；整届文件标为 partial。403 等访问拒绝会终止该届采集。刷新缺失摘要时，存储层保留原来已知的论文摘要及其来源观察信息，后续页面恢复时可以更新。
+
+针对旧 catalogue 的 blocked 届次可单独补采到新目录，避免与现有采集任务同时改写状态文件：
+
+```sh
+paper-atlas collect cvf --cache /data/paper-index/raw/http-cache \
+  --retry-catalogue /data/paper-index/collected/cvf-index-status.json \
+  --out /data/paper-index/collected/cvf-repair
+```
+
+parser_version 2 的清单还保存实际遍历的 index_urls、入口错误及缺失摘要页面。补采结果经过清单与身份核验后才导入，不能因重试任务启动就标记缺口已解决。
+
+## PMLR 中的 ICML 多轨道论文
+
+ICML 2024 起的一个 PMLR 卷可以同时包含研究主轨道与 Position Paper，不能把整卷都标为 `main`。采集器 v3 按轨道输出 `pmlr-v<VOLUME>-main.jsonl`、`pmlr-v<VOLUME>-position.jsonl` 及各自 manifest；卷级 `pmlr-v<VOLUME>.inventory.json` 和目录清单的 `files` 数组列出实际文件。导入队列应展开 `files`，不能假定每卷只有一个 `file`。
+
+2025、2026 年通过出版社元数据的真实 OpenReview forum ID，匹配官方完整节目单中的 `Conference` / `Position_Paper_Track` 来源组；忽略负编号生成的口头报告副本，遇到冲突、缺少身份或未匹配的发表记录停止发布结果。每条记录保存节目单 URL、SHA-256、事件 ID、关联身份及轨道证据。
+
+2024 年的官方节目单把两轨都放在 `Conference`，但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则；同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`：只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合，不进行全库模糊匹配。
+
+导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联；仅重新导入拆分文件会留下旧的主轨道事件，不能作为修复方式。完成迁移前，不把新文件直接导入现有库。

docs/development.md

@@ -0,0 +1,32 @@
+# 开发说明
+
+## 模块边界
+
+| 目录 | 职责 |
+| --- | --- |
+| `paper_atlas/cli.py` | 注册命令并延迟加载对应模块，统一参数入口 |
+| `paper_atlas/collectors/` | 各来源的发现、解析与清单输出，不直接写论文主库 |
+| `paper_atlas/io/` | 公共 HTTP、原始快照、HTML 树与受限 Git 元数据读取 |
+| `paper_atlas/ingestion/` | 核对清单、事务导入、维护锁和来源身份回填 |
+| `paper_atlas/db/` | 论文与投稿身份解析、来源保留、合并审计和 schema |
+| `paper_atlas/audits/` | 入库前来源对照及全部目标的覆盖报告 |
+| `paper_atlas/querying.py` | 一致性只读快照上的查询 |
+| `paper_atlas/resources/` | 随安装包提供的候选会议目录 |
+
+采集器依赖公共 IO，导入器依赖存储层，查询和审计读取既有证据。不要让公共网络模块反向依赖某个采集器，也不要在 CLI 中嵌入来源解析逻辑。新增采集器需要注册命令、输出带来源/届次/track 的清单，并补充有意义的来源边界验证。
+
+## 远程验证
+
+所有测试、构建、打包及安装验证在确认的 `ces-control:host` 环境进行。本机只编辑源码和进行不触发构建的检查。
+
+```sh
+python3 -B -m unittest discover -s tests -t . -v
+```
+
+`.gitea/workflows/check.yml` 会运行回归测试，构建并安装 wheel，然后从源码目录之外检查 console 命令、模块入口、schema 和会议目录资源。测试中只使用独立临时库，不连接运行中的论文主库。
+
+## 目录迁移
+
+原根目录脚本已迁入对应模块，统一使用 `paper-atlas <command>` 或 `python -m paper_atlas <command>`。不再把脚本文件路径作为接口。SQL schema、来源 JSONL、manifest 及既有数据库结构未因这次封装改变。
+
+部署使用固定 commit。仍在运行的旧批次继续使用已部署版本，待该批次完成后再切换后续任务；不要为了目录迁移重启已完成的全量导入。

docs/operations.md

@@ -0,0 +1,122 @@
+# 运行维护与数据核验
+
+## PostgreSQL 迁移
+
+共享实例的容器名为 `foundry-postgres`，底层镜像为 `postgres:17-alpine`；Compose 服务名仍为 `postgres`。数据库和 schema 是可配置的逻辑边界，表按研究数据域组织，不按调用项目复制。其他项目可获授 `research` 的 USAGE 及必要表权限，复用相同论文 ID 和表。未来换库只更新 libpq service；换 schema 可更新角色 search_path 或查询的 `--pg-schema`。
+
+复用已确认的现有实例，在共享 `foundry_data` 数据库按数据域使用 `research` schema 和专用角色，不修改其他业务库。安装、测试和构建在已批准的 `ces-control:host` 执行环境运行。libpq service 文件由服务器提供，权限必须为 `0600`，不得放进仓库或带密码的命令行参数。
+
+冻结 SQLite 写入，确认维护进程实际结束、WAL 已清空；采集到独立文件不修改索引。迁移持有源维护锁和目标 advisory lock，还须确保没有绕过协调器的旧版手工写入。
+
+```sh
+export PGSERVICEFILE=/etc/paper-atlas/pg_service.conf
+paper-atlas maintain postgres \
+  --db /data/paper-index/papers.sqlite --pg-service foundry-research \
+  --storage-path /var/lib/docker/volumes/endoleap-dev_postgres/_data \
+  --min-free-gib 8 --out /data/paper-index/collected/postgres-migration.json
+```
+
+全部 21 张业务表保留原 ID、原始 JSON 文本、NULL、来源、投稿、决定、标签出处和合并记录。`sqlite_sequence` 用于重设 PostgreSQL identity 序列，并考虑已合并的历史 ID，避免复用；SQLite FTS5 shadow 表根据完整标题/摘要重建为原生 GIN 索引，不逐字复制引擎文件。
+
+SQLite 允许文本包含 `U+0000`（NUL），PostgreSQL `text` 不允许。迁移使用 `nul-soh-v1` 可逆编码：NUL 存为 `U+0001 U+0001`，原有 `U+0001` 存为 `U+0001 U+0002`，其他字符不变；包括字串前缀在内的二进制排序保持一致，主键、唯一约束和外键可继续验证。编码版本保存在 `<schema>_migration.run.text_codec`，发布时同时写入 schema 注释 `paper-atlas:text-codec=nul-soh-v1`，其他项目只需研究表的权限即可读取格式信息，无须访问迁移控制表。CLI 的只读连接自动还原结果并编码筛选参数；直接 SQL/ETL 调用者应使用 `paper_atlas.postgres.text.encode/decode`，不能删除或直接替换控制字符。全文检索把这些控制字符当作分隔符，不承担逐字节匹配。
+
+先 COPY 到独立暂存 schema，每张表提交和记录哈希。中断后以同一命令、同一报告路径恢复，必须仍是**完全相同的未变更快照**。不同输入、已发布目标、未清空 WAL、新的未知业务表都会拒绝覆盖；恢复只会清空本次运行尚未完成且尚未发布的暂存表。保留错误现场，不盲目从头复制。
+
+恢复旧版本未发布的暂存数据时，在一个事务中转义已提交表里原有的 `U+0001`，然后记录新编码版本；失败回滚，重复恢复不会二次编码。已经提交的表无需从 SQLite 重新复制。报告中的 `source_sha256`、`target_sha256` 比较完整还原后的原始值；`target_storage_sha256` 另记实际存储值的哈希，包含转义的表两种哈希可以不同。
+
+发布前逐表按主键完整回读，比较数量及逐行 SHA-256；建立并验证所有主键、唯一约束、CHECK 和外键，重设序列、建立 GIN 并 ANALYZE。全部通过后原子重命名为 `research` schema，报告才为 `complete`。`copying`、`prepared`、部分表通过或 CI 通过均不代表生产迁移完成，也不替代原任务最后的实体去重和会议覆盖审计。
+
+发布后核对 PostgreSQL 的统计、全文检索、论文详情，以及 reject 后另会 accept 的历史。确认后保留 SQLite 只读备份，并在旁保存 `<SQLite路径>.postgresql-cutover.json`，记录报告路径、源哈希、目标 service/schema；新版本旧写入入口见此标记会拒绝写入。标记不含密码。后续导入适配属于下一阶段，不可继续更新 SQLite 后声称 PostgreSQL 已同步。
+
+远程集成测试通过 `PAPER_ATLAS_TEST_PG_SERVICE` 显式指定，使用随机前缀 schema 并仅清理自己的 fixture；没有该配置时会跳过，应另外报告“未运行”，不能用 SQLite 测试代替 PostgreSQL 验证。
+
+## PostgreSQL 增量写入
+
+`import records`、`import arxiv`、`maintain backfill` 使用 `--pg-service` 选择主库，`--pg-schema` 可覆盖配置的 search_path。写入只接受已经发布、拥有完整业务表和受支持编码标记的 schema，不自动建库或引导空表。SQLite 样本仍使用 `--db`，已切换备份继续禁止写入。
+
+写入连接按数据库/schema 获取会话级 advisory lock，文件提交、回滚以及 arXiv 分批提交均不释放；关闭连接才释放，其他写入者立即得到明确的占用错误。只读查询继续使用独立一致性快照。锁是所有调用方必须遵守的协议，其他项目不要绕过接口并发修改同一论文和投稿；[PostgreSQL 会话锁说明](https://www.postgresql.org/docs/17/explicit-locking.html#ADVISORY-LOCKS)解释了其事务边界。
+
+论文身份匹配、跨来源合并、投稿/决定来源和标签维护复用同一套领域逻辑。PostgreSQL 使用原生 RETURNING 和 identity 序列分配 ID，继承迁移时设定的历史 ID 高水位；失败和冲突尝试可以留下序列空洞，不能把最大 ID 当作行数。绑定文本按 `nul-soh-v1` 编码，结果解码，GIN 随论文写入自动更新。
+
+会议文件的数据变化在同一事务内完成，manifest 数量不符等错误会回滚整份文件，`ingest_runs` 另保留 failed、已处理数和零写入数；身份回填整个队列一次提交。arXiv 每 500 行提交数据和 checkpoint，异常只回滚尚未提交部分，重试必须仍为同一压缩包。大批次开始前核对独立审计清单、当前主库冲突和磁盘空间；完成后对数量、实际来源/决定关联及检索做数据验收，代码测试成功不代表生产导入成功。
+
+`audit public-openreview --pg-service ...` 在只读快照中核验两个受支持公开负面决定来源的固定 revision、逐文件哈希、manifest、原始出处标记和既有同届投稿。最新决定排序与查询端一致：观察时间、决定日期（为空时回退观察时间）、ID。实际导入持有写锁后再次复核该投稿的最新状态，阻止过期预检或弱来源标签覆盖冲突决定；冲突文件整体回滚并等待进一步证据核查，其他会议录用不构成同一次投稿的冲突。
+
+会议导入按原始字节流计算 SHA-256，在提交前与 manifest 提供的 `file_sha256` 比较；不符则回滚整份文件。成功批次的 checkpoint 会记录实际读取内容的哈希，便于复核导入时使用的文件版本。
+
+代码以固定 commit 部署，已经启动的任务继续使用其原版本。目录重构不要求迁移数据库，也不更改已有 JSONL 和 manifest 格式。
+
+## arXiv 原始编号与类别证据
+
+原始行数、唯一 arXiv 编号数和合并后的论文实体数分别核验。以下只读原始压缩包，流式处理到 ZIP EOF/CRC 检查完成；输出审计 JSON 与编号 SQLite 索引，不修改论文主库：
+
+```sh
+paper-atlas audit arxiv --archive /data/paper-index/raw/arxiv-metadata.zip \
+  --out /data/paper-index/collected/arxiv-source-audit.json
+```
+
+同编号的重复行会记录出现位置、次数和内容哈希版本，不直接视作不同论文；同篇论文的多个 arXiv 编号则保留各自来源及类别证据。旧导入器使用共享类别证据键，可能在论文实体合并时丢失某个编号的相同类别出处。新导入使用逐记录证据键；已有数据须在没有其他主库写入任务时按保留的来源 metadata 修复：
+
+```sh
+paper-atlas maintain arxiv-tags --db /data/paper-index/papers.sqlite \
+  --out /data/paper-index/collected/arxiv-tag-repair.json
+```
+
+修复使用同一维护锁，分批提交并保存断点，保留 `custom:*` 和其他来源证据；失败后确认原因已解决，再增加 `--resume` 从上次提交位置继续。完成报告不能作为重新扫描的断点，重新核验应指定新报告。源审计和标签修复均保留 8 GiB 空闲空间，不能代替所有来源入库后的实体、投稿去重与完整性检查。
+
+修复结束后还须核对每条 arXiv 来源记录的类别集合，不能只看旧证据键是否清零。下面的只读审计比较来源 metadata 中的类别、所属论文实体和逐记录证据键，同时检查没有合法 arXiv 来源的类别行：
+
+```sh
+paper-atlas audit arxiv-tags --db /data/paper-index/papers.sqlite \
+  --repair-report /data/paper-index/collected/arxiv-tag-repair.json \
+  --out /data/paper-index/collected/arxiv-tag-coverage-audit.json
+```
+
+审计持有共享维护锁；修复或其他协调写入尚未结束时会拒绝运行。提供修复报告时，还会核对其已完成状态、数据库路径和固定来源范围。审计以一致的只读快照分批读取，记录完整扫描数量、缺失/多余证据、错误论文归属、残留旧键及少量异常样例，不修改主库。
+
+来源记录按整数主键区间读取，避免 SQLite 误选按 external_id 排列的来源索引、每批重新排序整个来源；其他来源交错在编号区间内时仍按 source_id 过滤。类别证据继续使用 source_record_id 索引批量读取。完整核验范围不因查询优化缩小。
+
+只有全量记录核对完毕且无差异时才标记 `passed`；存在差异为 `failed` 并返回非零退出码，解析或执行中断为 `error`。运行中的报告不能作为完成证明。重新核验须使用新输出路径，不能覆盖已有报告；该检查不替代最后的实体去重、外键、SQLite 与全文索引核验。
+
+## 长期维护与更新
+
+1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发，遵守来源限流、重试与人机验证要求。
+2. 下载先写 `.part`，确认完整 HTTP 响应和 ZIP 目录，再记录 SHA-256 与时间后切换为正式文件。正式导入会读至 EOF，从而验证成员 CRC。
+3. 每次导入保留来源证据和批次，按会议/年份/track 报告覆盖状态。缺失摘要留空，不编造文本；缺失决定保留未知。
+4. 核验行数、唯一标识、外键、全文查询和 SQLite 完整性后，再将批次视为交付。样本通过不能证明全量正确。
+5. 数据库备份使用 SQLite backup API 或一致性快照，不能在写入过程中只复制 `.sqlite` 主文件而遗漏 WAL。按明确保留策略轮换备份，避免无上限积累。
+6. 不为每次数据更新生成新 Docker 镜像；不常驻下载器；不自动创建定时任务。本项目的更新任务应按需或经用户授权调度。
+7. 代码通过工作分支和 PR 交付；构建或 CI 只能使用用户确认的远程 runner，不在本机触发构建。
+
+## 当前访问限制
+
+2026-11-06 从目标服务器测试：Kaggle 元数据 GET 可下载；OpenReview 会议 group 接口可读，但 notes 接口返回 `ChallengeRequiredError`。这限制了直接补全；已使用来源可追溯的公开历史快照补充近期投稿与明确决定。仍不能用录用名单替代全部投稿，也不能声称覆盖了全部拒稿。
+
+## 查询批次与入库状态
+
+采集文件、已入库的来源记录和去重论文实体是不同的计数。来源可能覆盖同一篇论文，同一篇论文也可能有多次投稿；不能把各批文件数量相加作为独立论文总数。实时统计与覆盖报告使用：
+
+```sh
+paper-atlas query --db /data/paper-index/papers.sqlite --stats
+paper-atlas coverage --db /data/paper-index/papers.sqlite \
+  --out /data/paper-index/collected/venue-coverage-current.json
+```
+
+manifest 和采集器状态清单用于确认文件是否齐全及是否缺摘要；`ingest_runs` 和协调器日志用于确认主库导入是否成功。官方程序、书目目录和公开投稿快照分别保留，未出现在某份清单不等于被拒稿；第三方快照也不能代表平台当前全量状态。代码测试成功不能代替最终全库去重、完整性和逐届覆盖核验。
+
+当前代码与合并状态以 [Pull Requests](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls) 为准；CI 结果以对应提交的 Actions 检查为准。工作分支交付后等待用户手动合并。合并代码不自动部署、不自动启动采集，也不代表数据批次已完成。
+## 修正已有 PMLR 投稿的轨道
+
+先用采集器 v3 把 ICML 多轨道卷采集到新目录，保留旧文件。以下命令默认只读生成迁移计划；`--inventory` 可重复指定，以完整卷为单位核对，不能只提供 Position 子集：
+
+```sh
+paper-atlas maintain pmlr-tracks --db /data/paper-index/papers.sqlite \
+  --inventory /data/paper-index/collected/pmlr-tracks-v3/pmlr-v235.inventory.json \
+  --inventory /data/paper-index/collected/pmlr-tracks-v3/pmlr-v267.inventory.json \
+  --inventory /data/paper-index/collected/pmlr-tracks-v3/pmlr-v306.inventory.json \
+  --out /data/paper-index/collected/pmlr-track-plan.json
+```
+
+检查报告后，在同一命令添加 `--apply` 并使用新的 `--out` 路径执行迁移。计划使用共享维护锁，实际迁移使用独占锁；已有 arXiv 类别修复或其他写入运行时会拒绝启动，不会停止或重启它们。一次事务内核对整个来源集合、原始题目/摘要/作者、forum 身份、已有事件及目标冲突，更新原投稿的 edition 和身份归属，回填官方轨道出处及逐轨道覆盖数，最后复查轨道与外键后提交。论文、来源、投稿、决定的 ID 及决定观察均保留，不重新创建事件、不做实体合并、不改论文正文元数据或标签。
+
+报告记录每个迁移事件的旧 edition、原身份、完整决定观察及新轨道证据。`planned` 是未执行的计划；`complete` 才表示事务已提交；`error` 表示执行失败。若进程在提交前后中断而报告停在 `prepared`，先对照主库和报告核查，不根据日志缺失自行重启任务。新路径重跑会重新核对当前数据库，已正确的轨道不会再次迁移。共享多个来源、重复目标事件或冲突决定会停止迁移，需另行核对，不能直接删掉其中一条。

docs/usage.md

@@ -0,0 +1,100 @@
+# 使用指南
+
+先在已确认的远程环境安装项目，使用 `paper-atlas`。从源码运行时，将命令前缀替换为 `python3 -B -m paper_atlas`，其余参数相同。数据目录位于仓库外。
+
+## PostgreSQL 查询
+
+在远程环境安装 `.[postgres]`，通过权限为 `0600` 的 libpq service 文件连接，不把密码放入 URL、命令行或仓库。
+
+```sh
+export PGSERVICEFILE=/etc/paper-atlas/pg_service.conf
+paper-atlas query --pg-service foundry-research --stats
+paper-atlas query --pg-service foundry-research --query '"language model"' --tag arxiv:category=cs.CL
+paper-atlas query --pg-service foundry-research --status rejected --outcome accepted --limit 20
+paper-atlas query --pg-service foundry-research --paper 123
+```
+
+使用只读一致性事务，支持论文 ID、会议别名、年份、轨道、标签、最新决定和跨会议录用筛选。全文检索使用 PostgreSQL `websearch_to_tsquery('english', ...)`，支持引号短语、`OR` 和减号排除；与 SQLite FTS5 的语法及分词结果不完全相同。JSON 来源列保留原始文本；转成 `jsonb` 前须确认原始值符合该类型的限制。
+
+CLI 自动还原迁移中的 NUL/SOH 文本转义，详情中的来源摘要和筛选参数保留原字符。直接 SQL 读取可能看到 `U+0001` 转义序列；跨项目 ETL 应按迁移元数据中的 `text_codec` 使用 `paper_atlas.postgres.text.encode/decode`。存储格式和无损哈希验收见[运行维护](operations.md#postgresql-迁移)。
+
+迁移与切换见[运行维护](operations.md#postgresql-迁移)。`import records`、`import arxiv` 和 `maintain backfill` 均支持 `--pg-service` 和可选 `--pg-schema`；与 `--db` 二选一。SQLite 导入仅用于未切换的索引或独立样本。尚未适配 PostgreSQL 的维护、覆盖审计命令仍可只读检查冻结备份，不能当作主库的当前状态。
+
+## 1. 下载固定快照
+
+`paper-atlas download --help` 说明大小、校验和与续传参数。各来源的固定版本与具体命令见[数据源说明](data-sources.md)。下载成功只表示原始文件已核验。
+
+## 2. 采集并核对清单
+
+```sh
+paper-atlas collect pmlr --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+paper-atlas collect cvf --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+paper-atlas collect ecva --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+paper-atlas catalogue dblp --help
+paper-atlas audit iclr --help
+paper-atlas collect public-openreview --help
+paper-atlas collect icml-public-archive --help
+paper-atlas audit public-openreview --help
+```
+
+采集输出 JSONL、manifest 和逐卷或逐年 catalogue。逐项检查 blocked、partial、缺失摘要及目标会议归属。DBLP 候选目录还需核验届次和 track，不能直接当作主会录用名单。
+
+其他会议的公开拒稿使用 `collect public-openreview`，再以 `audit public-openreview` 核对既有投稿身份和决定冲突。来源版本、逐文件状态盘点和未解决记录分别保存在 inventory 与 held 清单中，具体命令见[数据源说明](data-sources.md#其他会议的公开拒稿)。这类公开子集不能当作会议完整投稿名单。
+
+主库切换后，预检须直接连接当前 PostgreSQL，不能用冻结的 SQLite 备份推断当前决定：
+
+```sh
+paper-atlas audit public-openreview --pg-service foundry-research \
+  --inventory /data/paper-index/collected/public-openreview-v1/public-openreview-inventory.json \
+  --out /data/paper-index/collected/public-negative-pg-preflight.json
+```
+
+预检使用只读一致性快照，报告会记录实际数据库、schema、文件哈希和同届次/轨道的决定冲突；其他会议的录用保留为独立历史。预检不预留写锁，实际导入这两类公开负面决定时会在写事务中再次复核同一次投稿的最新状态；过期预检报告不能覆盖已经变化的决定。
+
+## 3. 串行导入
+
+PostgreSQL 主库逐文件导入已经核验的资料：
+
+```sh
+paper-atlas import records --pg-service foundry-research \
+  /data/paper-index/collected/selected-file.jsonl
+paper-atlas maintain backfill --pg-service foundry-research \
+  --queue /data/paper-index/collected/audited-identity-queue.json
+```
+
+文件和 queue 须替换为实际核验通过的路径。会议导入按文件提交，失败时该文件的论文、标签和决定更改全部回滚，失败批次日志保留；重跑会按来源标识与决定证据去重。回填使用已有源记录，只合并经独立身份佐证的投稿，不重新导入文本。所有写入者须遵守同一数据库/schema 的会话锁；其他项目也应通过相同的写入接口更新共享论文和投稿。
+
+未来需要导入新的 arXiv 元数据包时使用 `paper-atlas import arxiv --pg-service foundry-research --archive ... --storage-path ...`。每 500 行提交断点，恢复同一文件时保留已提交 ID 和标签证据；`--storage-path` 指定运行主机上要监测的文件系统，在数据库主机执行时应指向数据库卷。已完成的历史压缩包不需要重复导入。
+
+SQLite 独立样本的首次 arXiv 导入使用 `paper-atlas import arxiv --db ... --archive ...`。SQLite 协调器仍适用于未切换的索引：
+
+```sh
+paper-atlas maintain merge --db /data/paper-index/samples/example.sqlite \
+  --archive /data/paper-index/raw/arxiv-metadata.zip \
+  /data/paper-index/collected/selected-file.jsonl
+```
+
+上例的 selected-file 须换成实际通过核验的文件。协调器会维护本项目的 arXiv 断点、序列化写入并检查外键；已完成的 arXiv 批次不会重启。持久批次应作为一次性 systemd 任务运行，详见[运行维护](operations.md)。
+
+## 4. 查询与分析前检查
+
+```sh
+paper-atlas query --db /data/paper-index/papers.sqlite --query '"language model"' --limit 20
+paper-atlas query --db /data/paper-index/papers.sqlite --tag arxiv:category=cs.AI
+paper-atlas query --db /data/paper-index/papers.sqlite --venue ICLR --year 2025 --status rejected
+paper-atlas query --db /data/paper-index/papers.sqlite --paper 123
+paper-atlas coverage --db /data/paper-index/papers.sqlite --out /data/paper-index/collected/coverage.json
+```
+
+`--status rejected` 筛选某次投稿的最新已知决定。它并不表示论文在所有会议均未录用；分析转投情况时查看该论文的完整投稿历史。年份表示会议届次，观察时间和决定日期分别保留。
+
+例如，查找在 ICLR 有拒稿记录、但在其他已知投稿中已有录用的论文：
+
+```sh
+paper-atlas query --db /data/paper-index/papers.sqlite \\
+  --venue ICLR --status rejected --outcome accepted
+```
+
+`known_acceptances` 列出目前仍为录用状态的各次投稿及会议、年份、track 和决定证据，`publication_outcome` 汇总为 accepted 或 no_known_acceptance。汇总会检查所有会议和 track，不把某次拒稿覆盖到整篇论文；同一次投稿的后续撤稿也会更新该次状态并保留原决定。no_known_acceptance 只表示现有来源中没有查到录用，不能称为“从未录用”。
+
+`--stats` 中 papers 是当前规则下的论文实体数；source_records 是来源记录数；decisions 是决定观察数。一个投稿可能有多个来源和多条观察，不能相加当作论文数。

import_arxiv.py

@@ -1,78 +0,0 @@
-"""Stream the official metadata ZIP into SQLite. No PDFs, source files or builds."""
-import argparse
-import json
-import time
-import shutil
-import zipfile
-from pathlib import Path
-
-
-from index_core import Store
-
-
-def ingest(db, archive):
-    store = Store(db)
-    conn = store.conn
-    conn.execute("INSERT OR IGNORE INTO sources(name,homepage) VALUES('arxiv','https://arxiv.org')")
-    source = conn.execute("SELECT id FROM sources WHERE name='arxiv'").fetchone()[0]
-    previous = conn.execute('SELECT id,checkpoint_json FROM ingest_runs WHERE source_id=? AND status IN (\'running\',\'failed\') ORDER BY id DESC LIMIT 1', (source,)).fetchone()
-    fingerprint = {'path':str(Path(archive).resolve()), 'bytes':Path(archive).stat().st_size, 'mtime_ns':Path(archive).stat().st_mtime_ns}
-    start = 0
-    if previous:
-        checkpoint = json.loads(previous[1] or '{}')
-        if checkpoint.get('archive') == fingerprint:
-            run = previous[0]
-            start = checkpoint.get('line', 0)
-            conn.execute("UPDATE ingest_runs SET status='running',error=NULL WHERE id=?", (run,))
-        else:
-            raise RuntimeError('Pending run belongs to a different archive; resolve it explicitly')
-    else:
-        run = conn.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)', (source,json.dumps({'archive':fingerprint,'line':0}))).lastrowid
-    conn.commit()
-    line_number = start
-    last = time.monotonic()
-    try:
-        with zipfile.ZipFile(archive) as zipped:
-            # Reading the member to EOF also checks its CRC.
-            with zipped.open('arxiv-metadata-oai-snapshot.json') as stream:
-                for line_number, line in enumerate(stream, 1):
-                    if line_number <= start:
-                        continue
-                    row = json.loads(line)
-                    external_id = row['id']
-                    title = ' '.join(row['title'].split())
-                    abstract = row.get('abstract', '').strip()
-                    authors = row.get('authors_parsed') or row.get('authors') or []
-                    metadata = {k:v for k,v in row.items() if k not in ('title','abstract','authors','authors_parsed')}
-                    paper,record = store.record('arxiv',external_id,'https://arxiv.org/abs/'+external_id,title,abstract,authors,{'arxiv':external_id,'doi':row.get('doi')},metadata,(row.get('versions') or [{}])[0].get('created'))
-                    conn.execute("DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key='arxiv:categories'", (record,))
-                    for category in (row.get('categories') or '').split():
-                        conn.execute("INSERT OR IGNORE INTO tags(namespace,value) VALUES('arxiv:category',?)",(category,))
-                        tag = conn.execute("SELECT id FROM tags WHERE namespace='arxiv:category' AND value=?",(category,)).fetchone()[0]
-                        conn.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,'arxiv:categories'))
-                    if line_number % 500 == 0:
-                        if shutil.disk_usage(Path(db).parent).free < 4*1024**3:
-                            raise RuntimeError('Stopped before exhausting disk: less than 4 GiB free')
-                        conn.execute('UPDATE ingest_runs SET records_seen=?,records_written=?,checkpoint_json=? WHERE id=?',(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
-                        conn.commit()
-                        if time.monotonic()-last >= 20:
-                            print(json.dumps({'run':run,'rows':line_number}),flush=True)
-                            last = time.monotonic()
-        conn.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
-        conn.commit()
-        print(json.dumps({'status':'complete','rows':line_number}),flush=True)
-    except BaseException as exc:
-        conn.rollback()
-        conn.execute("UPDATE ingest_runs SET status='failed',error=? WHERE id=?",(str(exc),run))
-        conn.commit()
-        raise
-    finally:
-        conn.close()
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--db', required=True)
-    parser.add_argument('--archive', required=True)
-    args = parser.parse_args()
-    ingest(args.db,args.archive)

paper_atlas/__init__.py

@@ -0,0 +1,3 @@
+"""Paper Atlas research metadata toolkit."""
+
+__version__ = '0.1.0'

paper_atlas/__main__.py

@@ -0,0 +1,3 @@
+from paper_atlas.cli import main
+
+raise SystemExit(main())

paper_atlas/audits/__init__.py

@@ -0,0 +1 @@
+"""Audits."""

paper_atlas/audits/arxiv.py

@@ -0,0 +1,90 @@
+"""Independently count raw arXiv IDs and their payload variants to ZIP EOF."""
+import argparse
+from contextlib import closing
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sqlite3
+import tempfile
+import zipfile
+
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.io.snapshots import digest_file
+
+MEMBER='arxiv-metadata-oai-snapshot.json'
+
+
+def fingerprint(path):
+    info=path.stat()
+    return {'bytes':info.st_size,'mtime_ns':info.st_mtime_ns}
+
+
+def audit(archive,out,expected_sha256=None):
+    out=Path(out).resolve();out.parent.mkdir(parents=True,exist_ok=True)
+    with out.with_suffix('.lock').open('a') as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        return _audit(archive,out,expected_sha256)
+
+
+def _audit(archive,out,expected_sha256=None):
+    archive=Path(archive).resolve();out=Path(out).resolve()
+    ids=out.with_suffix('.ids.sqlite')
+    if out.exists() or ids.exists():raise ValueError('Audit output exists; inspect it before another run')
+    out.parent.mkdir(parents=True,exist_ok=True)
+    before=fingerprint(archive);identity=digest_file(archive)
+    if expected_sha256 and identity['sha256']!=expected_sha256:raise ValueError('Archive checksum mismatch')
+    if shutil.disk_usage(out.parent).free<8*1024**3:raise RuntimeError('Less than 8 GiB free before source audit')
+    count=0;missing_title=0;missing_abstract=0
+    with tempfile.TemporaryDirectory(prefix='.arxiv-audit-',dir=out.parent) as temporary:
+        stage=Path(temporary)/ids.name
+        with closing(sqlite3.connect(stage)) as c:
+            c.executescript('PRAGMA cache_size=-16384; CREATE TABLE raw_ids(external_id TEXT PRIMARY KEY,first_line INTEGER,last_line INTEGER,occurrences INTEGER,first_sha256 BLOB,has_payload_variants INTEGER) WITHOUT ROWID; CREATE TABLE duplicate_variants(external_id TEXT,payload_sha256 BLOB,first_line INTEGER,last_line INTEGER,occurrences INTEGER,PRIMARY KEY(external_id,payload_sha256)) WITHOUT ROWID;')
+            with zipfile.ZipFile(archive) as zipped:
+                info=zipped.getinfo(MEMBER)
+                with zipped.open(info) as stream:
+                    for count,line in enumerate(stream,1):
+                        row=json.loads(line);external=row.get('id')
+                        if not isinstance(external,str) or not external or external!=external.strip():raise ValueError('Missing or malformed arXiv ID')
+                        payload=hashlib.sha256(json.dumps(row,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()).digest()
+                        inserted=c.execute('INSERT OR IGNORE INTO raw_ids VALUES(?,?,?,1,?,0)',(external,count,count,payload)).rowcount
+                        if not inserted:
+                            first_line,first=c.execute('SELECT first_line,first_sha256 FROM raw_ids WHERE external_id=?',(external,)).fetchone()
+                            c.execute('INSERT OR IGNORE INTO duplicate_variants VALUES(?,?,?,?,1)',(external,first,first_line,first_line))
+                            c.execute('INSERT INTO duplicate_variants VALUES(?,?,?,?,1) ON CONFLICT(external_id,payload_sha256) DO UPDATE SET last_line=excluded.last_line,occurrences=duplicate_variants.occurrences+1',(external,payload,count,count))
+                            c.execute('UPDATE raw_ids SET last_line=?,occurrences=occurrences+1,has_payload_variants=has_payload_variants OR first_sha256!=? WHERE external_id=?',(count,payload,external))
+                        missing_title+=not bool((row.get('title') or '').strip())
+                        missing_abstract+=not bool((row.get('abstract') or '').strip())
+                        if count%10000==0:
+                            c.commit()
+                            if shutil.disk_usage(out.parent).free<8*1024**3:raise RuntimeError('Less than 8 GiB free during source audit')
+                        if count%100000==0:print(json.dumps({'source':'arxiv_audit','raw_rows':count}),flush=True)
+                # zipfile verifies the member CRC only when its stream reaches EOF.
+            if not count:raise ValueError('Empty arXiv metadata member')
+            if fingerprint(archive)!=before:raise ValueError('Archive changed during audit')
+            c.commit()
+            if c.execute('PRAGMA quick_check').fetchone()[0]!='ok':raise ValueError('Audit index failed SQLite quick_check')
+            unique=c.execute('SELECT count(*) FROM raw_ids').fetchone()[0]
+            duplicate_ids=[]
+            for key,first,last,occurrences,variants in c.execute('SELECT external_id,first_line,last_line,occurrences,has_payload_variants FROM raw_ids WHERE occurrences>1 ORDER BY external_id'):
+                duplicate_ids.append({'external_id':key,'first_line':first,'last_line':last,'occurrences':occurrences,'has_payload_variants':bool(variants),
+                                      'variants':[{'sha256':digest.hex(),'first_line':a,'last_line':b,'occurrences':n} for digest,a,b,n in c.execute('SELECT payload_sha256,first_line,last_line,occurrences FROM duplicate_variants WHERE external_id=? ORDER BY first_line',(key,))]})
+            result={'checked_at':timestamp(),'archive':str(archive),'archive_identity':identity,'member':MEMBER,'member_crc32':f'{info.CRC:08x}',
+                    'uncompressed_bytes':info.file_size,'zip_eof_crc_verified':True,'raw_rows':count,'unique_external_ids':unique,'extra_rows_for_repeated_ids':count-unique,
+                    'repeated_id_count':len(duplicate_ids),'repeated_ids_with_payload_variants':sum(item['has_payload_variants'] for item in duplicate_ids),
+                    'missing_title_rows':missing_title,'missing_abstract_rows':missing_abstract,'duplicate_ids':duplicate_ids,'id_index':str(ids),
+                    'interpretation':'Raw ID/payload audit, not a count of unique research works. Payload variants are retained as evidence; latest source import uses the last row for each ID. Cross-source entity/submission dedup requires a separate database audit.'}
+        stage.replace(ids)
+        part=out.with_suffix('.part');part.write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');part.replace(out)
+    print(json.dumps({key:value for key,value in result.items() if key!='duplicate_ids'}),flush=True)
+    return result
+
+
+def main(argv=None,*,prog=None):
+    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--archive',required=True);parser.add_argument('--out',required=True)
+    parser.add_argument('--expected-sha256');args=parser.parse_args(argv)
+    audit(args.archive,args.out,args.expected_sha256)
+
+
+if __name__=='__main__':main()

paper_atlas/audits/arxiv_tags.py

@@ -0,0 +1,143 @@
+"""Compare every arXiv record's category evidence with retained source metadata."""
+import argparse
+from collections import defaultdict
+from contextlib import closing
+import fcntl
+import json
+from pathlib import Path
+import sqlite3
+
+from paper_atlas.db.submissions import timestamp
+
+
+def save(path, state):
+    part = path.with_suffix('.part')
+    part.write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n')
+    part.replace(path)
+
+
+def audit(db, out, repair_report=None, batch_size=500):
+    db, out = Path(db).resolve(), Path(out).resolve()
+    if not db.is_file():
+        raise ValueError('Existing database required')
+    if not 1 <= batch_size <= 500:
+        raise ValueError('Batch size must be between 1 and 500')
+    if out == db or repair_report and out == Path(repair_report).resolve():
+        raise ValueError('Audit output must preserve its inputs')
+    out.parent.mkdir(parents=True, exist_ok=True)
+    with Path(str(db) + '.maintenance.lock').open('a') as maintenance, Path(str(out) + '.lock').open('a') as output_lock:
+        # A partial repair is not a useful full-category audit. This also keeps
+        # cooperating maintenance writers out for the entire read-only snapshot.
+        fcntl.flock(maintenance, fcntl.LOCK_SH | fcntl.LOCK_NB)
+        fcntl.flock(output_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        if out.exists():
+            raise ValueError('Audit report exists; select a new output path')
+        with closing(sqlite3.connect(db.as_uri() + '?mode=ro', uri=True)) as c:
+            c.execute('PRAGMA cache_size=-32768')
+            c.execute('BEGIN')
+            source = c.execute("SELECT id FROM sources WHERE name='arxiv'").fetchone()
+            if source is None:
+                raise ValueError('No imported arXiv source')
+            total, maximum = c.execute('SELECT count(*),COALESCE(max(id),0) FROM source_records WHERE source_id=?', source).fetchone()
+            if not total:
+                raise ValueError('No imported arXiv records')
+            scope = {'db': str(db), 'source_id': source[0], 'source_record_count': total,
+                     'maximum_source_record_id': maximum}
+            if repair_report:
+                repair = json.loads(Path(repair_report).read_text())
+                if repair.get('status') != 'complete' or repair.get('records_refreshed') != total:
+                    raise ValueError('Category repair has not completed this source inventory')
+                if any(repair.get(key) != value for key, value in scope.items()):
+                    raise ValueError('Repair report belongs to a different source inventory')
+            state = {**scope, 'started_at': timestamp(), 'status': 'running', 'records_checked': 0,
+                     'last_record_id': 0, 'expected_evidence': 0, 'actual_evidence': 0,
+                     'records_with_mismatches': 0, 'missing_evidence': 0, 'unexpected_evidence': 0,
+                     'legacy_evidence': 0, 'examples': [], 'repair_report': str(Path(repair_report).resolve()) if repair_report else None}
+            save(out, state)
+            try:
+                while True:
+                    # The (source_id,external_id) index orders by external ID,
+                    # so choosing it sorts the entire source again per batch.
+                    # This fixed ID interval must use the integer primary key.
+                    records = c.execute('''SELECT id,external_id,paper_id,metadata_json FROM source_records NOT INDEXED
+                        WHERE source_id=? AND id>? AND id<=? ORDER BY id LIMIT ?''',
+                        (source[0], state['last_record_id'], maximum, batch_size)).fetchall()
+                    if not records:
+                        break
+                    ids = [row[0] for row in records]
+                    actual = defaultdict(set)
+                    for record, paper, category, evidence in c.execute('''SELECT pt.source_record_id,pt.paper_id,t.value,pt.evidence_key
+                        FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE t.namespace='arxiv:category'
+                        AND pt.source_record_id IN (''' + ','.join('?' for _ in ids) + ')', ids):
+                        actual[record].add((category, paper, evidence))
+                        state['legacy_evidence'] += evidence == 'arxiv:categories'
+                    for record, external, paper, payload in records:
+                        metadata = json.loads(payload)
+                        if not isinstance(metadata, dict):
+                            raise ValueError('arXiv source metadata must be an object')
+                        categories = metadata.get('categories')
+                        if categories is None:
+                            categories = ''
+                        if not isinstance(categories, str):
+                            raise ValueError('arXiv source categories must be text')
+                        evidence_key = f'arxiv:record:{record}:tags'
+                        expected = {(category, paper, evidence_key) for category in categories.split()}
+                        found = actual[record]
+                        missing, unexpected = expected - found, found - expected
+                        state['expected_evidence'] += len(expected)
+                        state['actual_evidence'] += len(found)
+                        state['missing_evidence'] += len(missing)
+                        state['unexpected_evidence'] += len(unexpected)
+                        if missing or unexpected:
+                            state['records_with_mismatches'] += 1
+                            if len(state['examples']) < 20:
+                                state['examples'].append({'record_id': record, 'external_id': external,
+                                    'paper_id': paper, 'missing': sorted(missing), 'unexpected': sorted(unexpected)})
+                        state['records_checked'] += 1
+                    state['last_record_id'] = ids[-1]
+                    if state['records_checked'] % 10000 < len(records):
+                        save(out, state)
+                        print(json.dumps({k: state[k] for k in ('status', 'records_checked', 'source_record_count', 'records_with_mismatches')}), flush=True)
+                if state['records_checked'] != total:
+                    raise ValueError('Audited record count differs from the fixed source inventory')
+                # Every category row must also have an arXiv source-record owner.
+                # The full namespace total catches orphan and wrong-source evidence
+                # that cannot appear in the per-record traversal above.
+                all_evidence = c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE t.namespace='arxiv:category'").fetchone()[0]
+                unscoped = all_evidence - state['actual_evidence']
+                if unscoped < 0:
+                    raise ValueError('Category evidence totals are inconsistent')
+                state.update(namespace_evidence=all_evidence, evidence_without_arxiv_record=unscoped)
+                if unscoped:
+                    state['unscoped_examples'] = [dict(zip(('paper_id', 'category', 'source_record_id', 'evidence_key'), row))
+                        for row in c.execute('''SELECT pt.paper_id,t.value,pt.source_record_id,pt.evidence_key
+                            FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id
+                            LEFT JOIN source_records r ON r.id=pt.source_record_id
+                            WHERE t.namespace='arxiv:category' AND (r.source_id IS NULL OR r.source_id<>?) LIMIT 20''', source)]
+                passed = not state['records_with_mismatches'] and not unscoped
+                state.update(status='passed' if passed else 'failed', finished_at=timestamp(),
+                             note='Full per-record category, canonical paper and evidence-key equality. '
+                                  'Read-only; not a global paper deduplication or full database-integrity audit.')
+                save(out, state)
+            except BaseException as error:
+                state.update(status='error', error=str(error), finished_at=timestamp())
+                save(out, state)
+                raise
+    print(json.dumps(state), flush=True)
+    return state
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--db', required=True)
+    parser.add_argument('--out', required=True)
+    parser.add_argument('--repair-report')
+    parser.add_argument('--batch-size', type=int, default=500)
+    args = parser.parse_args(argv)
+    result = audit(args.db, args.out, args.repair_report, args.batch_size)
+    if result['status'] != 'passed':
+        raise SystemExit('arXiv category evidence audit failed; inspect the report')
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/audits/coverage.py

@@ -1,10 +1,11 @@
 """Show every target venue, including those with no imported records."""
 import argparse
+from paper_atlas.config import DEFAULT_REGISTRY
 import json
 from pathlib import Path
 import sqlite3
 
-from submission_evidence import timestamp
+from paper_atlas.db.submissions import timestamp
 
 
 def report(db,registry):
@@ -32,10 +33,14 @@ def report(db,registry):
     finally:c.close()
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser();parser.add_argument('--db',required=True)
-    parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')));parser.add_argument('--out')
-    args=parser.parse_args();result=report(args.db,args.registry);text=json.dumps(result,ensure_ascii=False,indent=2)
+def main(argv=None, *, prog=None):
+    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--db',required=True)
+    parser.add_argument('--registry',default=str(DEFAULT_REGISTRY));parser.add_argument('--out')
+    args=parser.parse_args(argv);result=report(args.db,args.registry);text=json.dumps(result,ensure_ascii=False,indent=2)
     if args.out:
         path=Path(args.out);stage=path.with_suffix('.part');stage.write_text(text+'\n');stage.replace(path)
     else:print(text)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/audits/iclr.py

@@ -5,8 +5,8 @@ import hashlib
 import json
 from pathlib import Path
 
-from index_core import author_keys,normalized
-from submission_evidence import identities_for_row,timestamp
+from paper_atlas.db.store import author_keys,normalized
+from paper_atlas.db.submissions import identities_for_row,timestamp
 
 
 def load(path,source,year):
@@ -85,7 +85,11 @@ def audit(root,years,report_path):
     return result
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser();parser.add_argument('--root',required=True);parser.add_argument('--report',required=True)
-    parser.add_argument('--year',type=int,action='append',required=True);args=parser.parse_args()
+def main(argv=None, *, prog=None):
+    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--root',required=True);parser.add_argument('--report',required=True)
+    parser.add_argument('--year',type=int,action='append',required=True);args=parser.parse_args(argv)
     audit(args.root,args.year,args.report)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/audits/iclr_legacy.py

@@ -0,0 +1,172 @@
+"""Reconcile the 2020 program with independently identified official papers."""
+import argparse
+from collections import Counter, defaultdict
+import copy
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import re
+import tempfile
+
+from paper_atlas.audits.iclr import load
+from paper_atlas.db.store import author_keys, normalized
+from paper_atlas.db.submissions import timestamp
+
+
+LEGACY_URL = 'https://iclr.cc/virtual_2020/papers.json'
+LEGACY_SHA256 = '2405f2da6e2641fa600762f7d7cdf7c9d27f4d0ae65799168f3aee46036d30d2'
+LEGACY_BYTES = 391687
+EXPECTED = 687
+# This one paper has three documented titles across the pinned official legacy
+# list, current official program, and public submission snapshot. The exception
+# also requires independent public-author agreement; it is not fuzzy matching.
+REVIEWED_TITLE_VARIANT = {
+    'forum': 'SJgzLkBKPB',
+    'program': 'Explain Your Move: Understanding Agent Actions Using Salient and Relevant Feature Attribution',
+    'legacy': 'Explain Your Move: Understanding Agent Actions Using Focused Feature Saliency',
+    'snapshot': 'Explain Your Move: Understanding Agent Actions Using Specific and Relevant Feature Attribution',
+}
+
+
+def reconcile(program, snapshot, legacy):
+    by_number = defaultdict(list)
+    for row in snapshot.values():
+        by_number[row['metadata'].get('source_submission_number')].append(row)
+    legacy_by_forum = {}
+    for note in legacy:
+        forum = note.get('forum')
+        content = note.get('content') or {}
+        if not isinstance(forum, str) or not re.fullmatch(r'[A-Za-z0-9_-]+', forum):
+            raise ValueError('Invalid legacy forum identity')
+        if note.get('id') != forum or content.get('iclr_id') != forum or forum in legacy_by_forum:
+            raise ValueError('Conflicting or repeated official legacy forum')
+        if not isinstance(content.get('title'), str) or not content['title'].strip() or not isinstance(content.get('authors'), list):
+            raise ValueError('Legacy paper lacks title or public authors')
+        legacy_by_forum[forum] = content
+    if len(program) != len(legacy_by_forum):
+        raise ValueError('Official program and legacy inventories differ')
+    title_counts = Counter(normalized(row['title']) for row in legacy_by_forum.values())
+    enriched = []
+    assigned = set()
+    numbers = set()
+    evidence_counts = Counter()
+    for original in program:
+        if any(original.get(k) != v for k, v in {'source': 'iclr_program', 'venue': 'ICLR', 'year': 2020, 'track': 'main', 'status': 'accepted'}.items()):
+            raise ValueError('Program record is outside the accepted 2020 main track')
+        metadata = original.get('metadata') or {}
+        number = metadata.get('program_source_id')
+        if metadata.get('program_identity_type') != 'submission' or type(number) is not int or number <= 0 or number in numbers:
+            raise ValueError('Missing or repeated program submission number')
+        numbers.add(number)
+        candidates = by_number[number]
+        if len(candidates) != 1:
+            raise ValueError('Submission number is absent or ambiguous in the public snapshot')
+        submission = candidates[0]
+        forum = submission['identifiers']['openreview']
+        if submission['status'] != 'accepted' or forum not in legacy_by_forum or forum in assigned:
+            raise ValueError('Snapshot acceptance and official forum inventory do not agree')
+        content = legacy_by_forum[forum]
+        shared_legacy = len(author_keys(original['authors']) & author_keys(content['authors']))
+        shared_snapshot = len(author_keys(original['authors']) & author_keys(submission['authors']))
+        checks = {'legacy_title': normalized(original['title']) == normalized(content['title']),
+                  'snapshot_title': normalized(original['title']) == normalized(submission['title']),
+                  'snapshot_abstract': bool(original.get('abstract')) and original['abstract'].strip() == (submission.get('abstract') or '').strip(),
+                  'two_snapshot_authors': shared_snapshot >= 2}
+        independent_snapshot = sum(checks[k] for k in ('snapshot_title', 'snapshot_abstract', 'two_snapshot_authors')) >= 2
+        checks['legacy_title_unique'] = title_counts[normalized(content['title'])] == 1
+        independent_legacy = checks['legacy_title'] and checks['legacy_title_unique'] and shared_legacy >= 1
+        reviewed_variant = (forum == REVIEWED_TITLE_VARIANT['forum']
+                            and normalized(original['title']) == normalized(REVIEWED_TITLE_VARIANT['program'])
+                            and normalized(content['title']) == normalized(REVIEWED_TITLE_VARIANT['legacy'])
+                            and normalized(submission['title']) == normalized(REVIEWED_TITLE_VARIANT['snapshot'])
+                            and shared_legacy >= 2 and shared_snapshot >= 2)
+        if not (independent_snapshot or independent_legacy or reviewed_variant):
+            raise ValueError('Submission number alone cannot establish forum identity: ' + original['external_id'])
+        own = original.get('identifiers') or {}
+        if own.get('openreview') not in (None, forum):
+            raise ValueError('Existing program forum conflicts with independent evidence')
+        row = copy.deepcopy(original)
+        row['identifiers'] = {**own, 'openreview': forum}
+        row['url'] = 'https://iclr.cc/virtual_2020/poster_' + forum + '.html'
+        row['metadata'].update(identity_state='forum', program_submission_number_verified=True,
+            program_identity_verification={'method': 'official_legacy_and_public_submission_crosscheck',
+                'legacy_url': LEGACY_URL, 'legacy_sha256': LEGACY_SHA256,
+                'original_program_url': original['url'], 'legacy_forum': forum,
+                'checks': checks, 'shared_legacy_authors': shared_legacy,
+                'shared_snapshot_authors': shared_snapshot, 'reviewed_title_variant': reviewed_variant})
+        enriched.append(row)
+        assigned.add(forum)
+        evidence_counts.update({'independent_snapshot': int(independent_snapshot),
+                                'independent_legacy': int(independent_legacy),
+                                'reviewed_title_variant': int(reviewed_variant)})
+    if assigned != legacy_by_forum.keys():
+        raise ValueError('Unmatched official legacy forum identities remain')
+    if assigned != {forum for forum, row in snapshot.items() if row['status'] == 'accepted'}:
+        raise ValueError('Accepted snapshot forum inventory differs from the official program')
+    return enriched, dict(evidence_counts)
+
+
+def audit(program_path, snapshot_path, legacy_path, out):
+    program_path, snapshot_path, legacy_path = map(lambda p: Path(p).resolve(), (program_path, snapshot_path, legacy_path))
+    raw = legacy_path.read_bytes()
+    if len(raw) != LEGACY_BYTES or hashlib.sha256(raw).hexdigest() != LEGACY_SHA256:
+        raise ValueError('Official legacy snapshot checksum mismatch')
+    legacy = json.loads(raw)
+    if not isinstance(legacy, list) or len(legacy) != EXPECTED:
+        raise ValueError('Official legacy count mismatch')
+    manifest = json.loads(program_path.with_suffix('.manifest.json').read_text())
+    if manifest.get('parser_version', 0) < 2 or any(manifest.get(k) != v for k, v in {'source': 'iclr_program', 'venue': 'ICLR', 'year': 2020, 'track': 'main', 'count': EXPECTED}.items()):
+        raise ValueError('Invalid collected program inventory')
+    program_bytes = program_path.read_bytes()
+    program = [json.loads(line) for line in program_bytes.splitlines()]
+    if len(program) != EXPECTED or len({r['external_id'] for r in program}) != EXPECTED:
+        raise ValueError('Program count or source identity mismatch')
+    snapshot, snapshot_manifest, snapshot_sha = load(snapshot_path, 'openreview_snapshot', 2020)
+    rows, checks = reconcile(program, snapshot, legacy)
+    out = Path(out).resolve()
+    if out / 'iclr-program-2020-main.jsonl' in (program_path, snapshot_path, legacy_path):
+        raise ValueError('Output must preserve the original input files')
+    out.mkdir(parents=True, exist_ok=True)
+    with (out / 'iclr-legacy.lock').open('a') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        if (out / 'iclr-2020-identity-audit.json').exists():
+            raise ValueError('Completed identity audit exists; choose a new output directory')
+        with tempfile.TemporaryDirectory(prefix='.iclr-legacy-', dir=out) as temporary:
+            stage = Path(temporary)
+            path = stage / 'iclr-program-2020-main.jsonl'
+            path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in rows))
+            digest = hashlib.sha256(path.read_bytes()).hexdigest()
+            manifest.update(parser_version=3, file=str(out / path.name), file_sha256=digest,
+                            missing_forum_count=0, legacy_identity_sha256=LEGACY_SHA256)
+            path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
+            backfill = {'files': [str(out / path.name)], 'file_sha256': {str(out / path.name): digest}}
+            snapshots = {'files': [str(snapshot_path)], 'file_sha256': {str(snapshot_path): snapshot_sha}}
+            result = {'year': 2020, 'checked_at': timestamp(), 'program_records': len(rows),
+                      'snapshot_records': len(snapshot), 'snapshot_status_counts': snapshot_manifest['submission_status_counts'],
+                      'legacy_url': LEGACY_URL, 'legacy_sha256': LEGACY_SHA256,
+                      'program_input_sha256': hashlib.sha256(program_bytes).hexdigest(),
+                      'corroboration_counts': checks, 'backfill_queue': backfill, 'snapshot_files': snapshots,
+                      'note': 'No database writes. Backfill existing program identities before importing the snapshot; source IDs, titles and abstracts are preserved.'}
+            (stage / 'iclr-2020-backfill-queue.json').write_text(json.dumps(backfill, indent=2) + '\n')
+            (stage / 'iclr-2020-snapshot-queue.json').write_text(json.dumps(snapshots, indent=2) + '\n')
+            report = stage / 'iclr-2020-identity-audit.json'
+            report.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
+            for child in sorted(stage.iterdir(), key=lambda child: child == report):
+                child.replace(out / child.name)
+    print(json.dumps(result), flush=True)
+    return result
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--program', required=True)
+    parser.add_argument('--snapshot', required=True)
+    parser.add_argument('--legacy', required=True)
+    parser.add_argument('--out', required=True)
+    args = parser.parse_args(argv)
+    audit(args.program, args.snapshot, args.legacy, args.out)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/audits/public_openreview.py

@@ -0,0 +1,110 @@
+"""Check curated negative assertions against existing submission decisions."""
+import argparse
+from contextlib import closing
+import hashlib
+import json
+from pathlib import Path
+
+from paper_atlas.collectors.public_openreview import SOURCE, REVISION
+from paper_atlas.collectors.icml_public_archive import SOURCE as ARCHIVE_SOURCE, REVISION as ARCHIVE_REVISION
+from paper_atlas.db.submissions import timestamp, latest_status
+from paper_atlas.db.backend import database_arguments, open_read_connection
+
+
+def audit(db, inventory, out, *, pg_service=None, pg_schema=None):
+    inventory = Path(inventory).resolve()
+    result = json.loads(inventory.read_text())
+    specs = {SOURCE: (REVISION, 'curated_public_status_assertion'),
+             ARCHIVE_SOURCE: (ARCHIVE_REVISION, 'public_submission_venueid')}
+    source = result.get('source')
+    if source not in specs or result.get('revision') != specs[source][0]:
+        raise ValueError('Unsupported public source inventory')
+    revision, evidence_kind = specs[source]
+    conflicts = []
+    seen = set()
+    checked = 0
+    existing_attempts = 0
+    files = []
+    checksums = {}
+    with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
+        if pg_service:
+            identity = c.execute('SELECT current_database(),current_schema()').fetchone()
+            database, actual_schema = identity[0], identity[1]
+        else:
+            database, actual_schema = str(Path(db).resolve()), None
+        for manifest in result['files']:
+            path = Path(manifest['file'])
+            if hashlib.sha256(path.read_bytes()).hexdigest() != manifest['file_sha256']:
+                raise ValueError('Collected file differs from pinned inventory')
+            if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest:
+                raise ValueError('Collected manifest differs from inventory')
+            count = 0
+            # Unicode paragraph/line separators are valid inside JSON strings;
+            # split physical byte lines, not Unicode text boundaries.
+            for line in path.read_bytes().splitlines():
+                row = json.loads(line)
+                if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')):
+                    raise ValueError('Record edition differs from inventory')
+                meta = row['metadata']
+                if meta.get('evidence_kind') != evidence_kind or meta.get('source_revision') != revision:
+                    raise ValueError('Missing explicit public source provenance')
+                if meta.get('original_decision_signature_verified') is not False:
+                    raise ValueError('Curated metadata must not claim original signatures')
+                if row['status'] not in ('rejected', 'desk_rejected', 'withdrawn'):
+                    raise ValueError('Unsupported negative assertion')
+                forum = row['identifiers']['openreview']
+                if forum in seen:
+                    raise ValueError('Repeated forum in normalized inventory')
+                seen.add(forum)
+                existing = c.execute('''SELECT s.id FROM submission_identifiers i
+                    JOIN submissions s ON s.id=i.submission_id JOIN editions e ON e.id=i.edition_id
+                    JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? AND e.year=? AND e.track=?
+                    AND i.scheme='openreview' AND i.value=?''',
+                    (row['venue'], row['year'], row['track'], forum)).fetchall()
+                if len(existing) > 1:
+                    raise ValueError('Multiple canonical attempts for one forum')
+                if existing:
+                    existing_attempts += 1
+                    latest = latest_status(c, existing[0][0])
+                    if latest and latest not in ('unknown', 'under_review', row['status']):
+                        conflicts.append({'forum': forum, 'venue': row['venue'], 'year': row['year'],
+                                          'track': row['track'], 'submission_id': existing[0][0],
+                                          'existing_status': latest, 'curated_status': row['status']})
+                count += 1
+            if count != manifest['count']:
+                raise ValueError('Collected record count differs from inventory')
+            checked += count
+            files.append(str(path))
+            checksums[str(path)] = manifest['file_sha256']
+    if checked != result['normalized_records']:
+        raise ValueError('Normalized inventory total differs from checked records')
+    report = {'source': source, 'revision': revision, 'database': database,
+              'engine': 'postgresql' if pg_service else 'sqlite', 'schema': actual_schema, 'pg_service': pg_service,
+              'inventory': str(inventory), 'checked_at': timestamp(), 'records_checked': checked,
+              'existing_same_edition_attempts': existing_attempts, 'conflicts': conflicts,
+              'status': 'blocked' if conflicts else 'ready_for_serial_import',
+              'files': [] if conflicts else files, 'file_sha256': {} if conflicts else checksums,
+              'note': 'Read-only preflight. Other-conference acceptances remain separate history; '
+                      'this is not an entity deduplication or complete rejection-coverage proof.'}
+    out = Path(out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    part = out.with_suffix('.part')
+    part.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
+    part.replace(out)
+    print(json.dumps(report), flush=True)
+    return report
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    database_arguments(parser)
+    parser.add_argument('--inventory', required=True)
+    parser.add_argument('--out', required=True)
+    args = parser.parse_args(argv)
+    report = audit(args.db, args.inventory, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
+    if report['status'] == 'blocked':
+        raise SystemExit('Conflicting existing decisions require review before import')
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/cli.py

@@ -0,0 +1,56 @@
+"""One command tree; adapters are imported only when their command is selected."""
+import argparse
+from importlib import import_module
+
+from paper_atlas import __version__
+
+
+COMMANDS = {
+    ('query',): ('paper_atlas.querying', {}, '查询论文、标签和投稿历史'),
+    ('coverage',): ('paper_atlas.audits.coverage', {}, '查看全部候选会议的覆盖情况'),
+    ('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
+    ('collect','pmlr'): ('paper_atlas.collectors.pmlr', {}, 'PMLR 官方元数据'),
+    ('collect','acl'): ('paper_atlas.collectors.acl', {}, 'ACL Anthology XML'),
+    ('collect','iclr-program'): ('paper_atlas.collectors.iclr', {}, 'ICLR 官方公开程序'),
+    ('collect','iclr-snapshot'): ('paper_atlas.collectors.openreview', {}, 'ICLR 公开投稿与决定快照'),
+    ('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
+    ('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
+    ('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
+    ('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
+    ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
+    ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
+    ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
+    ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
+    ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
+    ('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'),
+    ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
+    ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
+    ('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
+    ('maintain','backfill'): ('paper_atlas.ingestion.backfill', {}, '回填既有投稿的来源身份'),
+    ('maintain','arxiv-tags'): ('paper_atlas.ingestion.arxiv_tags', {}, '恢复逐来源记录的 arXiv 类别证据'),
+    ('maintain','pmlr-tracks'): ('paper_atlas.ingestion.pmlr_tracks', {}, '核对并迁移既有 PMLR 投稿轨道'),
+    ('maintain','postgres'): ('paper_atlas.postgres.migration', {}, '迁移 SQLite 完整业务数据并核验 PostgreSQL'),
+    ('audit','iclr'): ('paper_atlas.audits.iclr', {}, '核对 ICLR 快照与同届官方程序'),
+    ('audit','iclr-legacy'): ('paper_atlas.audits.iclr_legacy', {}, '独立核验 ICLR 2020 旧目录投稿身份'),
+    ('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
+    ('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
+    ('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
+}
+
+
+def main(argv=None):
+    parser=argparse.ArgumentParser(prog='paper-atlas',description='Paper Atlas · 论文元数据与公开投稿历史')
+    parser.add_argument('--version',action='version',version='paper-atlas '+__version__)
+    commands=parser.add_subparsers(dest='command',required=True);groups={}
+    for route,(module,kwargs,help_text) in COMMANDS.items():
+        if len(route)==1:
+            leaf=commands.add_parser(route[0],help=help_text,add_help=False)
+        else:
+            if route[0] not in groups:
+                group=commands.add_parser(route[0],help={'collect':'采集来源元数据','catalogue':'盘点历史目录','import':'导入数据库','maintain':'维护数据库','audit':'核验数据'}[route[0]])
+                groups[route[0]]=group.add_subparsers(dest='operation',required=True)
+            leaf=groups[route[0]].add_parser(route[1],help=help_text,add_help=False)
+        leaf.set_defaults(handler=(route,module,kwargs))
+    args,forwarded=parser.parse_known_args(argv)
+    route,module,kwargs=args.handler
+    return import_module(module).main(forwarded,prog='paper-atlas '+' '.join(route),**kwargs)

paper_atlas/collectors/__init__.py

@@ -0,0 +1 @@
+"""Collectors."""

paper_atlas/collectors/acl.py

@@ -5,7 +5,7 @@ import re
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
-from metadata_checkout import checkout_metadata
+from paper_atlas.io.git_metadata import checkout_metadata
 
 VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'}
 
@@ -100,12 +100,12 @@ def collect_xml(path,out,commit):
     return results
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser()
+def main(argv=None, *, prog=None):
+    parser=argparse.ArgumentParser(prog=prog)
     parser.add_argument('--checkout',required=True)
     parser.add_argument('--out',required=True)
     parser.add_argument('--collection',action='append')
-    args=parser.parse_args()
+    args=parser.parse_args(argv)
     commit=checkout_metadata('https://github.com/acl-org/acl-anthology.git',args.checkout,['/data/xml/'])
     out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
     status_file=out/'acl-collection-status.json'
@@ -129,3 +129,7 @@ if __name__=='__main__':
     part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file)
     (out/'acl-last-run.json').write_text(json.dumps({'metadata_commit':commit,'collection_filter':args.collection,'blocked_collections':errors},indent=2))
     if errors:raise SystemExit(1)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/collectors/dblp.py

@@ -11,8 +11,8 @@ from pathlib import Path
 import sqlite3
 from xml.parsers import expat
 
-from download_snapshot import digest_file
-from submission_evidence import timestamp
+from paper_atlas.io.snapshots import digest_file
+from paper_atlas.db.submissions import timestamp
 
 
 XML_SHA='0376ad019b216553faff0a14e1757b2d4c7a3b84c1beacf0511ef83695211b18'
@@ -20,7 +20,7 @@ DTD_MD5='96e4283e71886dec2173718957293648'
 DTD_NAMES={'dblp.dtd','dblp-2023-06-28.dtd'}
 RELEASE='https://doi.org/10.4230/dblp.xml.2026-11-01'
 # Broad discovery candidates, including historical names; not a venue mapping.
-FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec'.split())
+FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
 
 
 def records(archive,dtd_bytes):
@@ -136,6 +136,10 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5):
         if c is not None:c.close()
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser();parser.add_argument('--archive',required=True);parser.add_argument('--dtd',required=True);parser.add_argument('--out',required=True)
-    args=parser.parse_args();catalogue(args.archive,args.dtd,args.out)
+def main(argv=None, *, prog=None):
+    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--archive',required=True);parser.add_argument('--dtd',required=True);parser.add_argument('--out',required=True)
+    args=parser.parse_args(argv);catalogue(args.archive,args.dtd,args.out)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/collectors/dblp_records.py

@@ -0,0 +1,198 @@
+"""Normalize pinned DBLP publication metadata into audited edition files."""
+import argparse
+from collections import Counter,OrderedDict
+from contextlib import closing
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import re
+import shutil
+import sqlite3
+import tempfile
+from urllib.parse import parse_qs,unquote,urlsplit
+
+from paper_atlas.config import DEFAULT_REGISTRY
+from paper_atlas.collectors.dblp import XML_SHA,DTD_MD5,RELEASE,records
+from paper_atlas.collectors.dblp_scope import scope
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.io.snapshots import digest_file
+from paper_atlas.io.publication_ids import mlsys_id
+
+AS_OF='2026-11-01T00:00:00.000Z'
+MATERIAL_PART=r'(?:title(?: page)?|preface|foreword|editorial|front\s?matter|back\s?matter|(?:table of )?contents|(?:author |subject )?index|contributors|(?:conference |program |organizing |steering )?committee|conference organization|reviewers)'
+FRONTMATTER=re.compile(MATERIAL_PART+r'(?:\s*[,;&]\s*'+MATERIAL_PART+r')*[.!:]?',re.I)
+HEADINGS=re.compile(r'(?:(?:IJCAI|AAAI)(?:[- ]\d{2,4})? |(?:19|20)\d{2} )?(?:conference )?(?:organization|organizers and sponsors|sponsorship|sponsors|(?:non-PC )?reviewers|awards and distinguished papers|keynote and invited speakers|area chairs and program committee|tutorials|panel session|invited talks|workshops|workshop summaries|special track on continuous planning)[.!:]?',re.I)
+
+
+def material_reason(title,authors,key,year,track):
+    if FRONTMATTER.fullmatch(title):return 'frontmatter'
+    if re.search(r"\bchairs['’]? welcome(?:\s*&\s*organization listing)?[.!:]?$",title,re.I):return 'frontmatter'
+    if not authors:
+        if HEADINGS.fullmatch(title) or re.fullmatch(r'(?:past IJCAI conferences|events colocated with IJCAI-\d{2}|welcome message from the conference chairs|welcome to (?:ICDM|IJCAI|AAAI) \d{4})[.!:]?',title,re.I):return 'proceedings_heading'
+        # These anonymous synthetic entries name whole workshop events, rather
+        # than identifiable papers. Hold them without discarding the raw source.
+        if year==2019 and track=='companion' and re.fullmatch(r'conf/www/X19[a-z]?',key):return 'unverified_proceedings_material'
+    return None
+
+
+def identifiers(fields):
+    dois=set();arxiv=set();forums=set();mlsys=set()
+    for link in fields.get('ee',[]):
+        if own_mlsys := mlsys_id(link):
+            mlsys.add(own_mlsys)
+        parsed=urlsplit(link);path=unquote(parsed.path)
+        doi=None
+        if parsed.netloc in ('doi.org','dx.doi.org','doi.ieeecomputersociety.org'):doi=path.lstrip('/')
+        elif parsed.netloc=='link.springer.com' and path.startswith('/chapter/'):doi=path.removeprefix('/chapter/')
+        elif parsed.netloc=='dl.acm.org':
+            match=re.match(r'/doi/(?:abs/|full/|pdf/)?(10\..+)',path)
+            if match:doi=match[1]
+        if doi and re.fullmatch(r'10\.\d{4,9}/\S+',doi):dois.add(doi.lower())
+        if parsed.netloc in ('arxiv.org','www.arxiv.org'):
+            match=re.fullmatch(r'/(?:abs|pdf)/((?:\d{4}\.\d{4,5}|[a-zA-Z.-]+/\d{7})(?:v\d+)?)(?:\.pdf)?',path)
+            if match:arxiv.add(re.sub(r'v\d+$','',match[1]))
+        if parsed.netloc=='openreview.net' and parsed.path in ('/forum','/pdf'):
+            values=parse_qs(parsed.query).get('id',[])
+            if len(values)==1 and re.fullmatch(r'[A-Za-z0-9_-]+',values[0]) and not re.match(r'20\d{2}-Oral-',values[0]):forums.add(values[0])
+    for doi in dois:
+        match=re.fullmatch(r'10\.48550/arxiv\.((?:\d{4}\.\d{4,5}|[a-z.-]+/\d{7})(?:v\d+)?)',doi)
+        if match:arxiv.add(re.sub(r'v\d+$','',match[1]))
+    publications={doi for doi in dois if not doi.startswith('10.48550/arxiv.')}
+    own={}
+    groups=(('doi',publications),('arxiv',arxiv),('openreview',forums),('publication_mlsys',mlsys))
+    for scheme,values in groups:
+        if len(values)==1:own[scheme]=next(iter(values))
+    ambiguous={scheme:sorted(values) for scheme,values in groups if len(values)>1}
+    return own,ambiguous
+
+
+def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026):
+    if row['record_type']!='inproceedings':return None,'not_a_paper'
+    fields=row['fields'];refs=fields.get('crossref',[])
+    if len(refs)!=1:return None,'ambiguous_or_missing_crossref'
+    edition,reason=scope(row,volumes.get(refs[0]),through_year)
+    if not edition:return None,reason
+    if not ({edition['venue']}|set(edition['venue_aliases']))&targets:return None,'outside_requested_venues'
+    if edition['track']=='workshop' and not include_workshops:return None,'separate_workshop'
+    titles=fields.get('title',[])
+    if len(titles)!=1 or not titles[0].strip():return None,'missing_or_multiple_titles'
+    title=' '.join(titles[0].split())
+    authors=[re.sub(r'\s+\d{4}$','',name).strip() for name in fields.get('author',[]) if name.strip()]
+    reason=material_reason(title,authors,row['key'],edition['year'],edition['track'])
+    if reason:return None,reason
+    own,ambiguous=identifiers(fields)
+    metadata={'snapshot_sha256':XML_SHA,'snapshot_release':RELEASE,'observation_date_precision':'day',
+              'proceedings_key':refs[0],'proceedings_title':edition['proceedings_title'],'bibliographic_year':edition['bibliographic_year'],
+              'edition_year_basis':edition['edition_year_basis'],
+              'scope_rule':edition['rule'],'author_names_raw':fields.get('author',[]),'publtype':row.get('publtype'),
+              'bibliography':{key:value for key,value in fields.items() if key in ('booktitle','year','ee','url','pages','volume','number','publisher','series','isbn')},
+              'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'}
+    if edition.get('track_scope_note'):
+        metadata['track_scope_note']=edition['track_scope_note']
+    flagged=bool(re.search(r'\b(?:withdrawn|retracted)\b',row.get('publtype') or '',re.I))
+    result={'source':'dblp','external_id':row['key'],'url':'https://dblp.org/rec/'+row['key']+'.html',
+            'title':title,'abstract':None,'authors':authors,'identifiers':own,
+            **{key:edition[key] for key in ('venue','venue_aliases','year','track')},
+            'status':'unknown' if flagged else 'accepted','observed_at':AS_OF,
+            'raw_decision':'listed in pinned DBLP inproceedings bibliography','metadata':metadata,
+            'tags':[{'namespace':'dblp:booktitle','value':fields['booktitle'][0]}]}
+    if flagged:result['decision_events']=[]
+    return result,None
+
+
+def collect(catalogue,out,venues=None,candidates=None,archive=None,dtd=None,include_workshops=False,through_year=2026):
+    if bool(candidates)==bool(archive):raise ValueError('Choose exactly one candidates file or original archive')
+    catalogue=Path(catalogue);manifest=json.loads(catalogue.with_name('dblp-ai-candidates.manifest.json').read_text())
+    if manifest['snapshot_sha256']!=XML_SHA:raise ValueError('Unsupported snapshot identity')
+    if candidates:
+        if digest_file(Path(candidates))['sha256']!=manifest['candidate_sha256']:raise ValueError('Candidate checksum mismatch')
+    else:
+        if not dtd or digest_file(Path(dtd))['md5']!=DTD_MD5:raise ValueError('DTD checksum mismatch')
+        if digest_file(Path(archive))['sha256']!=XML_SHA:raise ValueError('Archive checksum mismatch')
+    with closing(sqlite3.connect(catalogue.resolve().as_uri()+'?mode=ro',uri=True)) as c:
+        volumes={key:json.loads(payload) for key,payload in c.execute('SELECT key,metadata_json FROM proceedings')}
+    if len(volumes)!=manifest['record_types']['proceedings']:raise ValueError('Catalogue inventory mismatch')
+    allowed={row['acronym'].casefold():row['acronym'] for row in json.loads(DEFAULT_REGISTRY.read_text())['venues']}
+    if venues and set(name.casefold() for name in venues)-allowed.keys():raise ValueError('Unknown registry venue')
+    targets={allowed[name.casefold()] for name in venues} if venues else set(allowed.values())
+    if candidates:
+        families = manifest.get('candidate_family_counts', {})
+        missing = [venue for venue, family in [('MM', 'mm'), ('MLSys', 'mlsys')]
+                   if venue in targets and not families.get(family)]
+        if missing:
+            raise ValueError('Candidate file predates requested families ' + ', '.join(missing) + '; use the original --archive')
+    out=Path(out);out.mkdir(parents=True,exist_ok=True)
+    with (out/'dblp.normalize.lock').open('a') as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        return _collect_rows(volumes,out,targets,candidates,archive,dtd,include_workshops,through_year,manifest)
+
+
+def _collect_rows(volumes,out,targets,candidates,archive,dtd,include_workshops,through_year,source_manifest):
+    counts=Counter();excluded=Counter();examples={};groups={};seen=set();handles=OrderedDict();scanned=0
+    def rows():
+        if candidates:
+            with Path(candidates).open() as stream:
+                for line in stream:yield json.loads(line)
+        else:yield from records(archive,Path(dtd).read_bytes())
+    if shutil.disk_usage(out).free<8*1024**3:raise RuntimeError('Less than 8 GiB free before normalization')
+    with tempfile.TemporaryDirectory(prefix='.dblp-normalize-',dir=out) as temporary:
+        stage=Path(temporary)
+        try:
+            for row in rows():
+                scanned+=1
+                record,reason=normalize_row(row,volumes,targets,include_workshops,through_year)
+                if record is None:
+                    excluded[reason]+=1
+                    label=reason+'|'+'/'.join(row.get('key','').split('/')[:2])+'|'+';'.join(row['fields'].get('booktitle',[]))
+                    if label not in examples:examples[label]={'count':0,'example_key':row.get('key')}
+                    examples[label]['count']+=1
+                else:
+                    key=record['external_id']
+                    if key in seen:raise ValueError('Repeated DBLP publication key')
+                    seen.add(key);group=(record['venue'],record['year'],record['track'])
+                    filename='dblp-'+re.sub(r'[^a-z0-9]+','-',group[0].casefold())+'-'+str(group[1])+'-'+group[2]+'.jsonl'
+                    if group not in groups:groups[group]={'filename':filename,'count':0,'unknown':0,'ambiguous_identifiers':0}
+                    info=groups[group];info['count']+=1;info['unknown']+=record['status']=='unknown';info['ambiguous_identifiers']+=bool(record['metadata']['ambiguous_identifiers'])
+                    if group not in handles:
+                        if len(handles)>=32:handles.popitem(last=False)[1].close()
+                        handles[group]=(stage/filename).open('a')
+                    handles.move_to_end(group);handles[group].write(json.dumps(record,ensure_ascii=False)+'\n');counts[record['venue']]+=1
+                if scanned%50000==0:
+                    if shutil.disk_usage(out).free<8*1024**3:raise RuntimeError('Less than 8 GiB free during normalization')
+                    print(json.dumps({'source':'dblp','scanned':scanned,'normalized':sum(counts.values())}),flush=True)
+        finally:
+            for handle in handles.values():handle.close()
+        expected=source_manifest['candidate_records'] if candidates else sum(source_manifest['record_types'].values())
+        if scanned!=expected:raise ValueError('Input count differs from verified catalogue')
+        if not seen:raise ValueError('No target publications were verified')
+        files=[]
+        for (venue,year,track),info in sorted(groups.items()):
+            path=out/info['filename'];checksum=digest_file(stage/info['filename'])['sha256']
+            item={'source':'dblp','parser_version':1,'venue':venue,'year':year,'track':track,'count':info['count'],
+                  'missing_abstract_count':info['count'],'unknown_status_count':info['unknown'],'ambiguous_identifier_count':info['ambiguous_identifiers'],
+                  'list_url':RELEASE,'status':'partial','file':str(path),'file_sha256':checksum,'snapshot_sha256':XML_SHA,
+                  'abstract_note':'DBLP provides no abstract; canonical abstracts must come from independently linked sources.'}
+            (stage/info['filename']).replace(path)
+            meta=path.with_suffix('.manifest.json');part=meta.with_suffix('.part');part.write_text(json.dumps(item,indent=2));part.replace(meta);files.append(item)
+        result={'source':'dblp','finished_at':timestamp(),'snapshot_sha256':XML_SHA,'input_mode':'candidates' if candidates else 'original_archive',
+                'requested_venues':sorted(targets),'include_workshops':include_workshops,'through_year':through_year,'scanned_records':scanned,
+                'normalized_records':len(seen),'records_by_venue':dict(counts),'excluded_by_reason':dict(excluded),'scope_review':examples,'files':files,
+                'input_candidate_sha256':source_manifest['candidate_sha256'] if candidates else None,
+                'note':'Classification covers requested recognizable bibliography entries, not an independent proof of complete venue histories. Unknown scopes remain in scope_review.'}
+        target=out/'dblp-normalization.json';part=target.with_suffix('.part');part.write_text(json.dumps(result,ensure_ascii=False,indent=2));part.replace(target)
+        queue={'files':[item['file'] for item in files],'file_sha256':{item['file']:item['file_sha256'] for item in files}}
+        target=out/'dblp-import-queue.json';part=target.with_suffix('.part');part.write_text(json.dumps(queue,indent=2));part.replace(target)
+        print(json.dumps({key:result[key] for key in ('scanned_records','normalized_records','records_by_venue','excluded_by_reason')}),flush=True)
+        return result
+
+
+def main(argv=None,*,prog=None):
+    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--catalogue',required=True);parser.add_argument('--out',required=True)
+    inputs=parser.add_mutually_exclusive_group(required=True);inputs.add_argument('--candidates');inputs.add_argument('--archive')
+    parser.add_argument('--dtd');parser.add_argument('--venue',action='append');parser.add_argument('--include-workshops',action='store_true')
+    parser.add_argument('--through-year',type=int,default=2026);args=parser.parse_args(argv)
+    collect(args.catalogue,args.out,args.venue,args.candidates,args.archive,args.dtd,args.include_workshops,args.through_year)
+
+
+if __name__=='__main__':main()

paper_atlas/collectors/dblp_scope.py

@@ -0,0 +1,106 @@
+"""Explicit bibliographic aliases; a directory family alone is not venue evidence."""
+import re
+
+
+# Ordered long/joint names precede their shorter forms.
+BOOKS = {
+    'aaai': [('AAAI/IAAI','AAAI+IAAI',['AAAI','IAAI']),('AAAI','AAAI',[])],
+    'ijcai': [('IJCAI/ECAI','ECAI+IJCAI',['ECAI','IJCAI']),('IJCAI','IJCAI',[])],
+    'ecai': [('ECAI','ECAI',[]),('AISB/GI (ECAI)','ECAI',[]),('AISB (ECAI)','ECAI',[])],
+    'nips': [('NeurIPS Datasets and Benchmarks','NeurIPS',[]),('NeurIPS','NeurIPS',[]),('NIPS','NeurIPS',[])],
+    'icml': [('ICML','ICML',[]),('ML','ICML',[])], 'iclr':[('ICLR','ICLR',[])],
+    'colm':[('COLM','COLM',[])], 'colt':[('COLT/EuroCOLT','COLT',[]),('COLT','COLT',[])],
+    'aistats':[('AISTATS','AISTATS',[])], 'uai':[('UAI','UAI',[])],
+    'clear2':[('CLeaR','CLeaR',[])],
+    'acl':[('COLING-ACL','ACL+COLING',['ACL','COLING']),('ACL/IJCNLP','ACL',[]),('ACL','ACL',[])],
+    'emnlp':[('EMNLP-CoNLL','CoNLL+EMNLP',['CoNLL','EMNLP']),('EMNLP/IJCNLP','EMNLP',[]),('EMNLP','EMNLP',[])],
+    'naacl':[('HLT/EMNLP','EMNLP',[]),('NAACL-HLT','NAACL',[]),('HLT-NAACL','NAACL',[]),('NAACL','NAACL',[]),('HLT','NAACL',[])],
+    'eacl':[('EACL','EACL',[])],
+    'coling':[('LREC/COLING','COLING+LREC',['COLING','LREC']),('COLING-ACL','ACL+COLING',['ACL','COLING']),('COLING','COLING',[])],
+    'lrec':[('LREC/COLING','COLING+LREC',['COLING','LREC'])],
+    'conll':[('CoNLL/LLL','CoNLL',[]),('CoNLL','CoNLL',[])],
+    'cvpr':[('CVPR','CVPR',[])], 'iccv':[('ICCVW','ICCV',[]),('ICCV','ICCV',[])],
+    'eccv':[('ECCV','ECCV',[])], 'kdd':[('KDD','KDD',[])], 'sigir':[('SIGIR','SIGIR',[])],
+    'www':[('WWW','WWW',[])], 'wsdm':[('WSDM','WSDM',[])], 'recsys':[('RecSys','RecSys',[])],
+    'cikm':[('CIKM','CIKM',[])], 'icdm':[('ICDM','ICDM',[])], 'sdm':[('SDM','SDM',[])],
+    'ecml':[('ECML','ECML',['ECML-PKDD']),('EWSL','EWSL',['ECML-PKDD'])],
+    'pkdd':[('ECML/PKDD','ECML-PKDD',[]),('ECML PKDD','ECML-PKDD',[]),('PKDD','PKDD',['ECML-PKDD'])],
+    'icra':[('ICRA','ICRA',[])], 'iros':[('IROS','IROS',[])],
+    'rss':[('Robotics: Science and Systems','RSS',[])], 'corl':[('CoRL','CoRL',[])],
+    'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])],
+    'kr':[('KR','KR',[])], 'atal':[('AAMAS','AAMAS',[])],
+    'icassp':[('ICASSP','ICASSP',[])],
+    'interspeech':[('INTERSPEECH','INTERSPEECH',[]),('EUROSPEECH','EUROSPEECH',['INTERSPEECH']),('ICSLP','ICSLP',['INTERSPEECH']),('ECST','ECST',['INTERSPEECH'])],
+    'ismir':[('ISMIR','ISMIR',[])], 'miccai':[('MICCAI','MICCAI',[])],
+    'fat':[('FAccT','FAccT',[]),('FAT*','FAccT',[]),('FAT','FAccT',[])],
+    'mm':[('ACM Multimedia','MM',['ACMMM','ACM MM'])],
+    'mlsys':[('MLSys','MLSys',[]),('SysML','MLSys',['SysML'])],
+}
+
+
+def scope(row,volume,through_year=2026):
+    fields=row['fields'];books=fields.get('booktitle',[])
+    if len(books)!=1:return None,'ambiguous_booktitle'
+    book=books[0];key=row.get('key','');parts=key.split('/')
+    family=parts[1] if len(parts)>=3 and parts[0]=='conf' else ''
+    if family not in BOOKS:return None,'outside_target_family'
+    if not volume:return None,'missing_proceedings'
+    title=' '.join(volume['fields'].get('title',[]))
+    if family=='clear2' and 'causal learning and reasoning' not in title.casefold():return None,'ambiguous_venue_name'
+    if family=='mlsys' and 'machine learning and systems' not in title.casefold():return None,'ambiguous_venue_name'
+    matched=None
+    for base,venue,aliases in BOOKS[family]:
+        match=re.fullmatch(re.escape(base)+r'(?:\s+\((.*)\)|\s+(.*)|,\s*(Vol\.\s*\d+))?',book,re.I)
+        if match:
+            matched=(base,venue,aliases,' '.join(value for value in match.groups() if value));break
+    if not matched:return None,'unclassified_booktitle'
+    base,venue,aliases,suffix=matched
+    volume_family=volume['key'].split('/')[1] if volume['key'].startswith('conf/') else ''
+    if volume_family!=family and not any(item[1]==venue for item in BOOKS.get(volume_family,[])):
+        return None,'crossref_venue_conflict'
+    year_match=re.match(r'conf/[^/]+/((?:19|20)\d{2})',volume['key'])
+    year_basis='proceedings key, checked against bibliographic year or explicit event year in proceedings title'
+    if year_match:year=int(year_match[1])
+    else:
+        # Old SIGIR/CIKM/IJCAI keys use two digits. Never guess their century.
+        short=re.fullmatch(r'conf/[^/]+/(\d{2})',volume['key'])
+        years=set(re.findall(r'\b(?:19|20)\d{2}\b',title))
+        if not short or len(years)!=1:return None,'unverified_edition_year'
+        year=int(next(iter(years)))
+        if year%100!=int(short[1]):return None,'unverified_edition_year'
+        year_basis='unique explicit event year in proceedings title, corroborated by two-digit proceedings key'
+    if year>through_year:return None,'future_edition'
+    if venue=='MLSys' and year<2019:return None,'nonarchival_predecessor'
+    bibliographic=fields.get('year',[])
+    if len(bibliographic)!=1 or not re.fullmatch(r'\d{4}',bibliographic[0]):return None,'ambiguous_bibliographic_year'
+    if int(bibliographic[0])!=year and not re.search(r'\b'+str(year)+r'\b',title):return None,'unverified_year_difference'
+    text=(suffix+' '+title).casefold();track='main'
+    if 'findings' in text:track='findings'
+    elif 'shared task' in text:track='shared_task'
+    elif 'student research' in text or 'workshop' in text or 'wokshop' in text or base in ('ICCVW','EWSL'):track='workshop'
+    elif ('demo track' in text and ('industrial track' in text or 'applied data science track' in text)):track='proceedings_unspecified'
+    elif 'demonstration' in text or re.search(r'\bdemos?\b',text):track='demo'
+    elif 'tutorial' in text:track='tutorial'
+    elif 'industry' in text or 'industrial track' in text or 'applied data science' in text:track='industry'
+    elif 'position paper' in text:track='position'
+    elif 'companion' in text:track='companion'
+    elif 'late-breaking' in text or 'late breaking' in text:track='late_breaking'
+    elif 'doctoral' in text:track='doctoral'
+    elif 'challenge' in text or 'competition' in text:track='competition'
+    elif 'bridge program' in text:track='bridge'
+    elif 'special interest tracks' in text:track='special_interest'
+    elif 'alternate track' in text:track='alternate'
+    elif base=='NeurIPS Datasets and Benchmarks':track='datasets_benchmarks'
+    elif suffix and not (re.fullmatch(r'(?:\d+|Vol\.\s*\d+|Volume \d+: (?:Long|Short) Papers|Long Papers|Short Papers|Selected Papers|Fall)',suffix,re.I)
+                         or suffix.casefold() in ('poster','posters','oral','spotlight')):return None,'unclassified_track'
+    elif suffix.casefold() in ('poster','posters') and venue not in ('ICLR','ACL','EMNLP','NAACL','EACL','COLING','CoNLL'):track='poster'
+    if venue=='AAAI+IAAI':track='proceedings_unspecified'
+    if venue=='AAMAS' and year<2002:return None,'predecessor_not_aamas'
+    result = {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]),
+              'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis}
+    if venue=='MM' and track=='main':
+        # A main MM volume also includes doctoral, tutorials, companion papers
+        # and other sections. DBLP XML omits the per-paper TOC section.
+        result['track']='proceedings_unspecified'
+        result['track_scope_note']='Main proceedings volume; per-paper section is absent from the bibliography. Do not infer a regular research track.'
+    return result,None

paper_atlas/collectors/iclr.py

@@ -9,9 +9,9 @@ import re
 from urllib.error import HTTPError
 from urllib.parse import parse_qs, urljoin, urlsplit
 
-from collect_pmlr import fetch
-from collect_proceedings import Document, clean
-from index_core import author_keys, normalized
+from paper_atlas.io.http import fetch
+from paper_atlas.io.html import Document, clean
+from paper_atlas.db.store import author_keys, normalized
 
 ROOT = 'https://iclr.cc/'
 SOURCE = 'iclr_program'
@@ -141,6 +141,10 @@ def parse_program(data, abstracts, year, index_url):
         external = f'{year}/{forum}' if forum else f'{year}/program-{identity[1]}'
         missing_identity += not bool(forum)
         page = urljoin(ROOT, primary.get('virtualsite_url') or f'/virtual/{year}/{str(primary.get("eventtype", "Poster")).lower()}/{primary["id"]}')
+        if year == 2020 and not primary.get('virtualsite_url'):
+            # Numeric modern event IDs do not identify legacy poster pages.
+            # The independently verified forum bridge supplies the paper URL.
+            page = index_url
         topics = sorted({str(row['topic']).strip() for row in group if row.get('topic')})
         keywords = sorted({str(keyword).strip() for row in group for keyword in row.get('keywords', []) if keyword})
         records.append({'source': SOURCE, 'external_id': external, 'url': page,
@@ -208,12 +212,12 @@ def collect_year(year, cache, out):
     return manifest
 
 
-def main():
-    parser = argparse.ArgumentParser()
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
     parser.add_argument('--cache', required=True)
     parser.add_argument('--out', required=True)
     parser.add_argument('--year', type=int, action='append')
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
     years = discover(fetch(ROOT, args.cache, 4*1024**2, delay=1).decode())
     if args.year and set(args.year)-{year for year, _ in years}:

paper_atlas/collectors/icml_public_archive.py

@@ -0,0 +1,125 @@
+"""ICML 2026 public submission roots with explicit Rejected_Submission IDs."""
+import argparse
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import re
+import shutil
+import tempfile
+
+from paper_atlas.collectors.openreview import date_ms, invitations, public, value
+from paper_atlas.collectors.public_openreview import download
+from paper_atlas.db.submissions import timestamp
+
+
+SOURCE = 'openreview_public_archive'
+REVISION = '1a1b9d973a8dfbf254f740fb8100a2fcf36757ea'
+AS_OF = '2026-08-08T12:03:26.000Z'
+URL = 'https://raw.githubusercontent.com/HemantKumar01/ICML-2026-data/' + REVISION + '/data/rejected.json'
+BLOB_SHA1 = 'cebdcddffe3eb2145d32d328db89a1626e3f85eb'
+BYTES = 1600856
+EXPECTED = 214
+DOMAIN = 'ICML.cc/2026/Conference'
+VENUEID = DOMAIN + '/Rejected_Submission'
+
+
+def normalize(note):
+    forum = note.get('id')
+    if not isinstance(forum, str) or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', forum) or note.get('forum') != forum:
+        raise ValueError('Not a public submission root with a real forum identity')
+    if not public(note) or note.get('ddate'):
+        raise ValueError('Submission is not explicitly public and live in the archive')
+    if note.get('domain') != DOMAIN or DOMAIN + '/-/Submission' not in invitations(note):
+        raise ValueError('Submission is outside the ICML 2026 conference track')
+    content = note.get('content') or {}
+    if value(content, 'venueid') != VENUEID:
+        raise ValueError('Missing explicit platform Rejected_Submission classification')
+    title, abstract = value(content, 'title'), value(content, 'abstract')
+    if not isinstance(title, str) or not title.strip():
+        raise ValueError('Submission lacks a title')
+    if abstract is not None and not isinstance(abstract, str):
+        raise ValueError('Submission abstract must be text')
+    authors = value(content, 'authors', []) or []
+    if not isinstance(authors, list):
+        raise ValueError('Unexpected public author format')
+    authors = [a.strip() for a in authors if isinstance(a, str) and a.strip() and '@' not in a
+               and not re.fullmatch(r'anonymous(?: authors?)?', a.strip(), re.I)]
+    tags = []
+    for field in ('keywords', 'primary_area'):
+        raw = value(content, field)
+        terms = raw if isinstance(raw, list) else re.split(r'[,;]', raw) if field == 'keywords' and isinstance(raw, str) else [raw]
+        tags.extend({'namespace': SOURCE + ':' + field, 'value': term.strip()}
+                    for term in terms if isinstance(term, str) and term.strip())
+    url = 'https://openreview.net/forum?id=' + forum
+    return {'source': SOURCE, 'external_id': 'icml/2026/' + forum, 'url': url,
+            'title': ' '.join(title.split()), 'abstract': abstract.strip() if abstract else None,
+            'authors': authors, 'identifiers': {'openreview': forum},
+            'submission_identifiers': [{'scheme': 'openreview', 'value': forum}],
+            'venue': 'ICML', 'year': 2026, 'track': 'main', 'status': 'rejected',
+            'raw_decision': VENUEID, 'evidence_url': url, 'observed_at': AS_OF,
+            'submitted_at': date_ms(note.get('cdate') or note.get('tcdate')), 'tags': tags,
+            'metadata': {'source_revision': REVISION, 'source_blob_sha1': BLOB_SHA1,
+                         'source_file': URL, 'original_forum': forum, 'license': note.get('license'),
+                         'venue': value(content, 'venue'), 'venueid': VENUEID,
+                         'public_readers_verified': True, 'original_invitations': invitations(note),
+                         'source_note_modified_at': date_ms(note.get('mdate')),
+                         'evidence_kind': 'public_submission_venueid',
+                         'original_decision_signature_verified': False,
+                         'source_snapshot_upper_bound': AS_OF}}
+
+
+def collect(out, fetcher=None):
+    fetcher = fetcher or download
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    with (out / 'collect.lock').open('a') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        if (out / 'public-openreview-inventory.json').exists():
+            raise ValueError('Completed inventory exists; select a new output directory')
+        if shutil.disk_usage(out).free < 8 * 1024**3:
+            raise RuntimeError('Less than 8 GiB free before collection')
+        data = fetcher(URL, BYTES)
+        blob = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
+        if len(data) != BYTES or blob != BLOB_SHA1:
+            raise ValueError('Public archive checksum mismatch')
+        archive = json.loads(data)
+        notes = archive['notes']
+        if archive.get('count') != EXPECTED or not isinstance(notes, list) or len(notes) != EXPECTED:
+            raise ValueError('Public archive inventory count mismatch')
+        records = [normalize(note) for note in notes]
+        if len({row['identifiers']['openreview'] for row in records}) != EXPECTED:
+            raise ValueError('Repeated submission forum in public archive')
+        records.sort(key=lambda row: row['external_id'])
+        with tempfile.TemporaryDirectory(prefix='.icml-public-', dir=out) as temporary:
+            stage = Path(temporary)
+            path = stage / 'public-openreview-icml-2026-main.jsonl'
+            path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
+            manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICML', 'year': 2026,
+                        'track': 'main', 'count': EXPECTED, 'status': 'partial', 'list_url': URL,
+                        'file': str(out / path.name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
+                        'source_revision': REVISION, 'missing_abstract_count': sum(not row['abstract'] for row in records),
+                        'evidence_kind': 'public_submission_venueid',
+                        'note': 'Public opt-in subset; explicit platform venueid, not original signed PC notes.'}
+            path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
+            result = {'source': SOURCE, 'revision': REVISION, 'finished_at': timestamp(),
+                      'input_rows': EXPECTED, 'normalized_records': EXPECTED, 'files': [manifest],
+                      'source_files': [{'url': URL, 'git_blob_sha1': blob, 'input_bytes': len(data)}],
+                      'note': 'Snapshot preserves per-note public visibility and licenses; no reviews, authorids or PDFs retained.'}
+            inventory = stage / 'public-openreview-inventory.json'
+            inventory.write_text(json.dumps(result, indent=2) + '\n')
+            for child in sorted(stage.iterdir(), key=lambda child: child == inventory):
+                child.replace(out / child.name)
+            print(json.dumps(result), flush=True)
+            return result
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--out', required=True)
+    args = parser.parse_args(argv)
+    collect(args.out)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/collectors/openreview.py

@@ -10,7 +10,7 @@ import sqlite3
 import tarfile
 import tempfile
 
-from download_snapshot import digest_file
+from paper_atlas.io.snapshots import digest_file
 
 
 SOURCE='openreview_snapshot'
@@ -194,7 +194,11 @@ def collect(archive,out,years,expected_sha=SHA256):
         return {year:catalogue[str(year)] for year in years}
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser();parser.add_argument('--archive',required=True);parser.add_argument('--out',required=True)
-    parser.add_argument('--year',type=int,action='append');args=parser.parse_args()
+def main(argv=None, *, prog=None):
+    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--archive',required=True);parser.add_argument('--out',required=True)
+    parser.add_argument('--year',type=int,action='append');args=parser.parse_args(argv)
     collect(args.archive,args.out,args.year or sorted(EXPECTED))
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/collectors/peerread.py

@@ -9,7 +9,7 @@ import urllib.request
 from collections import defaultdict
 from pathlib import Path
 
-from collect_pmlr import fetch
+from paper_atlas.io.http import fetch
 
 
 def collect(cache,out):
@@ -63,9 +63,13 @@ def collect(cache,out):
         print(json.dumps(manifest),flush=True)
 
 
-if __name__=='__main__':
-    parser = argparse.ArgumentParser()
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
     parser.add_argument('--cache',required=True)
     parser.add_argument('--out',required=True)
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     collect(args.cache,args.out)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/collectors/pmlr.py

@@ -1,17 +1,16 @@
 """Collect official PMLR metadata, never the PDFs linked by the metadata."""
 import argparse
+from collections import defaultdict
 import hashlib
-from http.client import IncompleteRead
+from paper_atlas.config import DEFAULT_REGISTRY
 import io
 import json
 import re
 import os
 import subprocess
-import ssl
 import tarfile
 import time
-import urllib.error
-import urllib.request
+import tempfile
 from pathlib import Path
 
 import yaml
@@ -40,45 +39,8 @@ def venue_key(shortname):
     return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper())
 
 
-def fetch(url,cache,limit=64*1024**2,delay=0.5):
-    cache = Path(cache)
-    cache.mkdir(parents=True,exist_ok=True)
-    path = cache/hashlib.sha256(url.encode()).hexdigest()
-    if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
-        if path.stat().st_size>limit:raise RuntimeError('Metadata response exceeds byte limit')
-        return path.read_bytes()
-    for attempt in range(3):
-        try:
-            request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
-            with urllib.request.urlopen(request,timeout=90) as response:
-                if response.status != 200:
-                    raise RuntimeError('Expected complete HTTP 200 response')
-                expected=int(response.headers.get('Content-Length','0'))
-                if expected>limit:
-                    raise RuntimeError('Metadata response exceeds byte limit')
-                data = response.read(limit+1)
-                if len(data)>limit:
-                    raise RuntimeError('Metadata response exceeds byte limit')
-                if expected and len(data)!=expected:
-                    raise IncompleteRead(data,max(0,expected-len(data)))
-            part = path.with_suffix('.part')
-            part.write_bytes(data)
-            part.replace(path)
-            path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
-            time.sleep(delay)
-            return data
-        except urllib.error.HTTPError as exc:
-            if exc.code not in (429,500,502,503,504) or attempt==2:
-                exc.close()
-                raise
-            retry_after=exc.headers.get('Retry-After','1');exc.close()
-            try:retry_after=int(retry_after)
-            except ValueError:retry_after=5
-            time.sleep(min(30,max(2**attempt,retry_after)))
-        except (urllib.error.URLError,TimeoutError,ConnectionError,IncompleteRead,ssl.SSLError) as exc:
-            reason=getattr(exc,'reason',exc)
-            if isinstance(reason,ssl.SSLCertVerificationError) or attempt==2:raise
-            time.sleep(min(30,2**(attempt+1)))
+from paper_atlas.io.http import fetch
+from paper_atlas.collectors.pmlr_tracks import collect_track_evidence, record_track
 
 
 def metadata_texts(volume,cache):
@@ -140,67 +102,98 @@ def collect_volume(volume,cache,out,allowed=None):
     track = 'workshop' if 'workshop' in title.lower() else 'main'
     path = Path(out)/f'pmlr-v{volume}.jsonl'
     path.parent.mkdir(parents=True,exist_ok=True)
-    part = path.with_suffix('.part')
+    mixed_icml = venue == 'ICML' and year >= 2024
+    mapping, track_program = collect_track_evidence(year, cache) if mixed_icml else ({}, None)
     records = {}
     duplicate_posts = 0
-    with part.open('w') as output:
-        for member_name,content in metadata_texts(volume,cache):
-            parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
-            if len(parts)<3 or parts[0].strip():
-                raise RuntimeError('Missing YAML frontmatter: '+member_name)
-            row = load_metadata(parts[1])
-            if row.get('layout') not in ('inproceedings','article'):
-                continue
-            if not row.get('title'):
-                raise ValueError('Paper metadata has no title: '+member_name)
-            if not row.get('id'):
-                raise ValueError('Paper metadata has no publication ID: '+member_name)
-            external = f"v{volume}/{row['id']}"
-            ids = {}
-            for key in ('doi','openreview','arxiv'):
-                value = row.get(key)
-                if value:
-                    value = str(value)
-                    if key=='arxiv':
-                        value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
-                    if key=='openreview':
-                        value = value.split('id=')[-1].split('&')[0]
-                    ids[key] = value
-            record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
-            version={'path':member_name,'date':row.get('date')}
-            if external in records:
-                previous=records[external]
-                if ({k:v for k,v in previous.items() if k!='metadata'} != {k:v for k,v in record.items() if k!='metadata'}
-                    or {k:v for k,v in previous['metadata'].items() if k not in ('date','source_versions')} != {k:v for k,v in record['metadata'].items() if k!='date'}):
-                    raise ValueError('Conflicting metadata posts share a publication ID: '+external)
-                previous['metadata']['source_versions'].append(version)
-                if str(row.get('date') or '')>str(previous['metadata'].get('date') or ''):
-                    previous['metadata']['date']=row['date']
-                duplicate_posts+=1
-                continue
-            record['metadata']['source_versions']=[version]
-            records[external]=record
-        for record in records.values():
-            output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
+    for member_name,content in metadata_texts(volume,cache):
+        parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
+        if len(parts)<3 or parts[0].strip():
+            raise RuntimeError('Missing YAML frontmatter: '+member_name)
+        row = load_metadata(parts[1])
+        if row.get('layout') not in ('inproceedings','article'):
+            continue
+        if not row.get('title'):
+            raise ValueError('Paper metadata has no title: '+member_name)
+        if not row.get('id'):
+            raise ValueError('Paper metadata has no publication ID: '+member_name)
+        external = f"v{volume}/{row['id']}"
+        ids = {}
+        for key in ('doi','openreview','arxiv'):
+            value = row.get(key)
+            if value:
+                value = str(value)
+                if key=='arxiv':
+                    value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
+                if key=='openreview':
+                    value = value.split('id=')[-1].split('&')[0]
+                ids[key] = value
+        record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
+        version={'path':member_name,'date':row.get('date')}
+        if external in records:
+            previous=records[external]
+            if ({k:v for k,v in previous.items() if k!='metadata'} != {k:v for k,v in record.items() if k!='metadata'}
+                or {k:v for k,v in previous['metadata'].items() if k not in ('date','source_versions')} != {k:v for k,v in record['metadata'].items() if k!='date'}):
+                raise ValueError('Conflicting metadata posts share a publication ID: '+external)
+            previous['metadata']['source_versions'].append(version)
+            if str(row.get('date') or '')>str(previous['metadata'].get('date') or ''):
+                previous['metadata']['date']=row['date']
+            duplicate_posts+=1
+            continue
+        record['metadata']['source_versions']=[version]
+        records[external]=record
     count=len(records)
     missing_abstracts=sum(not (record.get('abstract') or '').strip() for record in records.values())
     if count==0:
-        part.unlink(missing_ok=True)
         raise RuntimeError('No papers parsed; refusing an empty successful collection')
-    part.replace(path)
-    result = {'source':'pmlr','parser_version':2,'venue':venue,'year':year,'track':track,'volume':volume,'count':count,'metadata_post_count':count+duplicate_posts,'duplicate_post_count':duplicate_posts,'missing_abstract_count':missing_abstracts,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'partial' if missing_abstracts else 'collected'}
-    path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
-    print(json.dumps(result),flush=True)
+    grouped = defaultdict(list)
+    for record in records.values():
+        if mixed_icml:
+            record_track(record, mapping, track_program)
+        grouped[record['track']].append(record)
+    # Parse and verify the entire volume before replacing any published output.
+    manifests = []
+    with tempfile.TemporaryDirectory(prefix=f'.pmlr-v{volume}-', dir=path.parent) as temporary:
+        stage = Path(temporary)
+        for current_track, rows in sorted(grouped.items()):
+            filename = f'pmlr-v{volume}-{current_track}.jsonl' if mixed_icml else path.name
+            output = stage / filename
+            output.write_text(''.join(json.dumps(row, ensure_ascii=False, default=str)+'\n' for row in rows))
+            absent = sum(not (row.get('abstract') or '').strip() for row in rows)
+            manifest = {'source': 'pmlr', 'parser_version': 3, 'venue': venue, 'year': year,
+                        'track': current_track, 'volume': volume, 'count': len(rows),
+                        'missing_abstract_count': absent, 'list_url': f'https://proceedings.mlr.press/v{volume}/',
+                        'file': str(path.parent / filename), 'file_sha256': hashlib.sha256(output.read_bytes()).hexdigest(),
+                        'status': 'partial' if absent else 'collected'}
+            if track_program:
+                manifest['track_program'] = track_program
+            output.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
+            manifests.append(manifest)
+        result = {'source': 'pmlr', 'parser_version': 3, 'venue': venue, 'year': year,
+                  'track': next(iter(grouped)) if len(grouped) == 1 else 'mixed', 'volume': volume,
+                  'count': count, 'metadata_post_count': count + duplicate_posts,
+                  'duplicate_post_count': duplicate_posts, 'missing_abstract_count': missing_abstracts,
+                  'list_url': f'https://proceedings.mlr.press/v{volume}/', 'files': manifests,
+                  'status': 'partial' if missing_abstracts else 'collected'}
+        if len(manifests) == 1:
+            result['file'] = manifests[0]['file']
+        if track_program:
+            result['track_program'] = track_program
+        inventory = stage / f'pmlr-v{volume}.inventory.json'
+        inventory.write_text(json.dumps(result, indent=2)+'\n')
+        for child in sorted(stage.iterdir(), key=lambda child: child == inventory):
+            child.replace(path.parent / child.name)
+    print(json.dumps(result), flush=True)
     return result
 
 
-def main():
-    parser = argparse.ArgumentParser()
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
     parser.add_argument('--cache',required=True)
     parser.add_argument('--out',required=True)
     parser.add_argument('--volume',type=int,action='append')
-    parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
-    args = parser.parse_args()
+    parser.add_argument('--registry',default=str(DEFAULT_REGISTRY))
+    args = parser.parse_args(argv)
     allowed = {venue_key(v['acronym']) for v in json.loads(Path(args.registry).read_text())['venues']}
     volumes = args.volume
     if not volumes:

paper_atlas/collectors/pmlr_tracks.py

@@ -0,0 +1,140 @@
+"""Resolve mixed ICML proceedings tracks from the official paper program."""
+from collections import Counter
+import hashlib
+import json
+import re
+from urllib.parse import parse_qs, urlsplit
+
+from paper_atlas.collectors.iclr import identifiers, source_number
+from paper_atlas.db.store import author_keys, normalized
+from paper_atlas.io.html import Document, clean
+from paper_atlas.io.http import fetch
+
+
+POSITION_LIST_POLICY = 'https://icml.cc/Conferences/2025/CallForPositionPapers'
+PUBLISHER_LINK_ALIASES = {'v235/theo24a': 'v235/guyard24a'}
+
+
+def position_list_policy(data):
+    """The organizers explicitly link this title filter as the 2024 position list."""
+    for node in Document(data.decode()).root.walk():
+        if node.tag != 'a' or clean(node.text()).casefold() != 'position papers':
+            continue
+        url = node.attrs.get('href', '')
+        parsed = urlsplit(url)
+        query = parse_qs(parsed.query)
+        if (parsed.netloc in ('', 'icml.cc') and parsed.path == '/virtual/2024/papers.html'
+                and query.get('filter') == ['titles'] and query.get('search') == ['position:']):
+            return {'url': POSITION_LIST_POLICY, 'sha256': hashlib.sha256(data).hexdigest(),
+                    'position_list_url': 'https://icml.cc' + parsed.path + '?' + parsed.query}
+    raise ValueError('Official policy no longer identifies the 2024 position-paper title list')
+
+
+def program_tracks(data, year, policy=None):
+    program = json.loads(data)
+    events = program.get('results') if isinstance(program, dict) else None
+    if (not isinstance(events, list) or program.get('count') != len(events)
+            or program.get('next') or program.get('previous')):
+        raise ValueError('Official ICML program must be a complete counted event list')
+    if len({str(row['id']) for row in events}) != len(events):
+        raise ValueError('Repeated event identity in official ICML program')
+    if year == 2024 and policy is None:
+        raise ValueError('The 2024 position-list policy is required')
+    mapping = {}
+    excluded = Counter()
+    for row in events:
+        number = source_number(row)
+        if number is not None and number < 0:
+            excluded['synthetic_oral_event'] += 1
+            continue
+        parsed = urlsplit(row.get('sourceurl') or '')
+        group = parse_qs(parsed.query).get('id', [])
+        groups = {f'ICML.cc/{year}/Conference': 'main',
+                  f'ICML.cc/{year}/Position_Paper_Track': 'position'}
+        if (parsed.netloc != 'openreview.net' or parsed.path != '/group'
+                or len(group) != 1 or group[0] not in groups):
+            excluded['other_source'] += 1
+            continue
+        if not str(row.get('decision') or '').casefold().startswith('accept'):
+            raise ValueError('Non-accepted event in official ICML paper program')
+        if year == 2024:
+            # The 2024 program flattens both tracks into Conference and links
+            # PMLR publication IDs instead of forums. Use only this year-specific,
+            # organizer-endorsed list rule, never a global title heuristic.
+            link = urlsplit(row.get('paper_pdf_url') or '')
+            if link.netloc != 'proceedings.mlr.press' or not re.fullmatch(r'/v\d+/[^/]+\.html', link.path):
+                raise ValueError('2024 ICML paper has no publisher self-link')
+            key = ('pmlr', link.path.lstrip('/').removesuffix('.html'))
+            track = 'position' if re.match(r'^Position(?: Paper)?:', row.get('name') or '', re.I) else 'main'
+            kind = 'official_2024_position_list'
+        else:
+            forum = identifiers(row).get('openreview')
+            if not forum:
+                raise ValueError('ICML paper has no real OpenReview self-identity')
+            key = ('openreview', forum)
+            track, kind = groups[group[0]], 'official_openreview_track_group'
+        evidence = {'track': track, 'kind': kind, 'event_ids': [row['id']],
+                    'source_group': group[0], 'program_title': row.get('name')}
+        if key[0] == 'pmlr' and key[1] in PUBLISHER_LINK_ALIASES:
+            evidence['publisher_link_alias'] = key[1]
+            evidence['_authors'] = [author['fullname'] for author in row.get('authors', [])]
+            key = ('pmlr', PUBLISHER_LINK_ALIASES[key[1]])
+        if key in mapping:
+            if mapping[key]['track'] != track:
+                raise ValueError('One paper identity has conflicting official tracks')
+            mapping[key]['event_ids'].append(row['id'])
+        else:
+            mapping[key] = evidence
+    if not mapping:
+        raise ValueError('Official ICML program contains no supported paper identities')
+    return mapping, {'event_count': len(events), 'paper_identity_count': len(mapping),
+                     'excluded_events': dict(excluded)}
+
+
+def collect_track_evidence(year, cache):
+    url = f'https://icml.cc/static/virtual/data/icml-{year}-orals-posters.json'
+    data = fetch(url, cache)
+    policy = position_list_policy(fetch(POSITION_LIST_POLICY, cache)) if year == 2024 else None
+    mapping, inventory = program_tracks(data, year, policy)
+    provenance = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), **inventory}
+    if policy:
+        provenance['position_list_policy'] = policy
+    return mapping, provenance
+
+
+def record_track(record, mapping, provenance):
+    key = ('pmlr', record['external_id']) if record['year'] == 2024 else ('openreview', record['identifiers'].get('openreview'))
+    if key not in mapping:
+        raise ValueError('Published ICML paper is absent from the verified official track inventory: ' + record['external_id'])
+    evidence = {**mapping[key], 'identity_scheme': key[0], 'identity_value': key[1],
+                'program_url': provenance['url'], 'program_sha256': provenance['sha256']}
+    if 'publisher_link_alias' in evidence:
+        overlap = author_keys(record.get('authors', [])) & author_keys(evidence.pop('_authors', []))
+        if normalized(record['title']) != normalized(evidence['program_title']) or len(overlap) < 2:
+            raise ValueError('Reviewed publisher-link alias lacks exact title and independent author corroboration')
+        evidence['corroborating_author_count'] = len(overlap)
+    if 'position_list_policy' in provenance:
+        evidence['position_list_policy'] = provenance['position_list_policy']
+    record['track'] = evidence['track']
+    record['metadata']['track_evidence'] = evidence
+
+
+def require_track_manifest(manifest):
+    if (manifest.get('source') == 'pmlr' and str(manifest.get('venue', '')).upper() == 'ICML'
+            and int(manifest.get('year', 0)) >= 2024
+            and (manifest.get('parser_version', 0) < 3 or not manifest.get('track_program'))):
+        raise ValueError('Recollect mixed ICML PMLR proceedings with parser_version>=3 and official track evidence')
+
+
+def require_track_record(row, manifest):
+    require_track_manifest({**manifest, **{key: row.get(key) for key in ('source', 'venue', 'year')}})
+    if manifest.get('track_program'):
+        evidence = row.get('metadata', {}).get('track_evidence', {})
+        program = manifest['track_program']
+        if (evidence.get('track') != row.get('track') or evidence.get('program_url') != program.get('url')
+                or evidence.get('program_sha256') != program.get('sha256')
+                or not evidence.get('event_ids') or evidence.get('identity_scheme') not in ('pmlr', 'openreview')):
+            raise ValueError('PMLR record differs from its official track evidence')
+        own = row['external_id'] if evidence['identity_scheme'] == 'pmlr' else row.get('identifiers', {}).get('openreview')
+        if not own or evidence.get('identity_value') != own:
+            raise ValueError('PMLR track evidence belongs to a different paper identity')

paper_atlas/collectors/proceedings.py

@@ -1,64 +1,21 @@
-"""Collect COLM/NeurIPS official abstract pages without downloading PDFs."""
+"""Collect COLM, NeurIPS and MLSys official metadata without downloading PDFs."""
 import argparse
 import json
 import re
 import time
 from collections import defaultdict
-from html.parser import HTMLParser
 from pathlib import Path
 from urllib.parse import urljoin,urlsplit
 
-from collect_pmlr import fetch
+from paper_atlas.io.http import fetch
+from paper_atlas.io.publication_ids import mlsys_id
 
-ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/'}
+ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/',
+       'mlsys':'https://proceedings.mlsys.org/'}
+VENUES={'colm':'COLM','neurips':'NeurIPS','mlsys':'MLSys'}
 
 
-class Node:
-    def __init__(self,tag='',attrs=None):
-        self.tag=tag
-        self.attrs={key:value or '' for key,value in attrs or []}
-        self.children=[]
-
-    def text(self):
-        if self.tag in ('script','style'):
-            return ''
-        content=''.join(child.text() if isinstance(child,Node) else child for child in self.children)
-        return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content
-
-    def walk(self):
-        yield self
-        for child in self.children:
-            if isinstance(child,Node):
-                yield from child.walk()
-
-
-class Document(HTMLParser):
-    VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'}
-
-    def __init__(self,html):
-        super().__init__(convert_charrefs=True)
-        self.root=Node()
-        self.stack=[self.root]
-        self.feed(html)
-
-    def handle_starttag(self,tag,attrs):
-        node=Node(tag,attrs)
-        self.stack[-1].children.append(node)
-        if tag not in self.VOID:
-            self.stack.append(node)
-
-    def handle_endtag(self,tag):
-        for index in range(len(self.stack)-1,0,-1):
-            if self.stack[index].tag==tag:
-                del self.stack[index:]
-                break
-
-    def handle_data(self,data):
-        self.stack[-1].children.append(data)
-
-
-def clean(text):
-    return ' '.join(text.split())
+from paper_atlas.io.html import Document,Node,clean
 
 
 def track_key(text):
@@ -118,6 +75,22 @@ def parse_paper(html,url,source,year,index_url):
     track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split())
     track=track_key(track_text or url)
     identifiers={}
+    if source == 'mlsys':
+        if year < 2019:
+            raise ValueError('SysML 2018 was non-archival; official proceedings begin in 2019')
+        own = mlsys_id(url)
+        if not own or int(own.split('/')[0]) != year:
+            raise ValueError('MLSys paper URL does not identify this publication year')
+        identifiers['publication_mlsys'] = own
+        if track_text:
+            explicit = {'conference': 'main', 'main track': 'main', 'research track': 'main',
+                        'industry track': 'industry', 'industrial track': 'industry',
+                        'mlsys' + str(year): 'main'}
+            if track_text.casefold() not in explicit:
+                raise ValueError('Unclassified official MLSys track: ' + track_text)
+            track = explicit[track_text.casefold()]
+        elif year >= 2025 and not url.endswith('-Abstract-Conference.html'):
+            raise ValueError('Modern MLSys paper lacks an explicit conference track')
     if meta['citation_doi']:
         identifiers['doi']=meta['citation_doi'][0]
     action_nodes=[]
@@ -136,8 +109,11 @@ def parse_paper(html,url,source,year,index_url):
             from urllib.parse import parse_qs
             forum=parse_qs(parsed.query).get('id')
             if forum: identifiers['openreview']=forum[0]
-    venue='COLM' if source=='colm' else 'NeurIPS'
-    return {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}}
+    venue=VENUES[source]
+    result = {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}}
+    if source == 'mlsys' and year == 2019:
+        result['venue_aliases'] = ['SysML']
+    return result
 
 
 def discover(source,cache):
@@ -188,13 +164,14 @@ def collect_index(source,index_url,year,cache,out):
     return results
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser()
-    parser.add_argument('--source',choices=ROOTS,required=True)
+def main(argv=None, *, prog=None, source=None):
+    parser=argparse.ArgumentParser(prog=prog)
+    if source is None:parser.add_argument('--source',choices=ROOTS,required=True)
+    else:parser.set_defaults(source=source)
     parser.add_argument('--cache',required=True)
     parser.add_argument('--out',required=True)
     parser.add_argument('--year',type=int,action='append')
-    args=parser.parse_args()
+    args=parser.parse_args(argv)
     out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
     status_path=out/f'{args.source}-index-status.json'
     catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
@@ -214,3 +191,7 @@ if __name__=='__main__':
         part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path)
     (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2))
     if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/collectors/public_openreview.py

@@ -0,0 +1,252 @@
+"""Collect explicit public negative labels without inferring unobserved decisions.
+
+PaperCopilot is a curated mirror, not the conference's signed decision feed.
+Only metadata is retained; source profiles, reviews and PDFs are never cached.
+"""
+import argparse
+from collections import Counter, defaultdict
+import fcntl
+import hashlib
+import io
+import json
+from pathlib import Path
+import re
+import shutil
+import tempfile
+import urllib.request
+
+from paper_atlas.config import DEFAULT_REGISTRY
+from paper_atlas.db.submissions import timestamp
+
+
+SOURCE = 'papercopilot_openreview'
+REVISION = '41ee277825d685bd78e88e48b69da6aead9a69bb'
+AS_OF = '2026-10-31T01:25:52.000Z'
+REPOSITORY = 'https://github.com/papercopilot/openreview'
+RAW_ROOT = 'https://raw.githubusercontent.com/papercopilot/openreview/' + REVISION + '/'
+TREE_URL = 'https://api.github.com/repos/papercopilot/openreview/git/trees/' + REVISION + '?recursive=1'
+NEGATIVE = {'reject': 'rejected', 'rejected': 'rejected',
+            'desk reject': 'desk_rejected', 'desk rejected': 'desk_rejected',
+            'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
+TRACKS = {'main': 'main', 'Position': 'position',
+          'Datasets & Benchmarks': 'datasets_benchmarks'}
+WORKSHOP_REJECT_LABELS = {"aim-fm workshop @ neurips'24 reject",
+                          'coginterp @ neurips 2025 reject'}
+
+
+def download(url, limit):
+    request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata)'})
+    with urllib.request.urlopen(request, timeout=90) as response:
+        if response.status != 200:
+            raise ValueError('Expected a complete HTTP 200 response')
+        result = response.read(limit + 1)
+    if len(result) > limit:
+        raise ValueError('Public metadata exceeds byte limit')
+    return result
+
+
+def negative_status(raw):
+    """A closed mapping: pending, absent, low-score and retracted are not Reject."""
+    if not isinstance(raw, str):
+        raise ValueError('Status must be text')
+    key = ' '.join(raw.casefold().split())
+    if key in NEGATIVE:
+        return NEGATIVE[key]
+    if re.fullmatch(r'(?:icml|neurips) 20\d{2} (?:conference|workshop [a-z0-9 -]+) withdrawn submission', key):
+        return 'withdrawn'
+    if key in WORKSHOP_REJECT_LABELS:
+        return 'rejected'
+    return None
+
+
+def normalize(row, family, year, provenance, include_workshops=False):
+    """Return a normalized assertion or a reason requiring separate review."""
+    raw_status = row.get('status', '')
+    status = negative_status(raw_status)
+    if status is None:
+        if re.search(r'\bretract\w*\b', raw_status, re.I):
+            return None, 'retracted'
+        if re.search(r'\b(?:reject\w*|withdraw\w*|desk)\b', raw_status, re.I):
+            return None, 'unmapped_negative_label'
+        return None, 'not_explicit_negative'
+    forum = row.get('id')
+    if not isinstance(forum, str) or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', forum) or re.match(r'20\d{2}-Oral-', forum):
+        raise ValueError('Invalid public submission forum ID')
+    track = row.get('track')
+    if not isinstance(track, str):
+        raise ValueError('Track must be explicit text')
+    normalized_track = TRACKS.get(track)
+    if normalized_track is None:
+        if re.fullmatch(r'workshop_[a-z0-9_]+', track) and include_workshops:
+            normalized_track = track
+        else:
+            return None, 'separate_workshop' if track.startswith('workshop') else 'unmapped_track'
+    prefixed = re.match(r'^(ICML|NeurIPS) (20\d{2}) ', raw_status)
+    if prefixed and (prefixed[1].casefold() != ('neurips' if family == 'nips' else family) or int(prefixed[2]) != year):
+        raise ValueError('Status label conflicts with source conference year')
+    allowed = {v['acronym'].casefold(): v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}
+    venue = allowed.get({'nips': 'neurips', 'acmmm': 'mm'}.get(family, family))
+    if venue is None:
+        return None, 'unmapped_venue'
+    title = row.get('title')
+    abstract = row.get('abstract')
+    if not isinstance(title, str) or not title.strip():
+        raise ValueError('Public submission lacks a title')
+    if abstract is not None and not isinstance(abstract, str):
+        raise ValueError('Abstract must be text or null')
+    author_text = row.get('author') or ''
+    if not isinstance(author_text, str):
+        raise ValueError('Unexpected public author format')
+    authors = [a.strip() for a in author_text.split(';') if a.strip() and '@' not in a
+               and not re.fullmatch(r'anonymous(?: authors?)?', a.strip(), re.I)]
+    tags = []
+    for field in ('keywords', 'primary_area'):
+        value = row.get(field)
+        if value is None:
+            continue
+        terms = value if isinstance(value, list) else re.split(r'[,;]', value) if field == 'keywords' and isinstance(value, str) else [value]
+        tags.extend({'namespace': SOURCE + ':' + field, 'value': term.strip()}
+                    for term in terms if isinstance(term, str) and term.strip())
+    evidence = provenance['source_file'] + '#L' + str(provenance['source_line'])
+    return {'source': SOURCE, 'external_id': f'{family}/{year}/{forum}',
+            'url': 'https://openreview.net/forum?id=' + forum,
+            'title': ' '.join(title.split()), 'abstract': abstract.strip() if abstract else None,
+            'authors': authors, 'identifiers': {'openreview': forum},
+            'submission_identifiers': [{'scheme': 'openreview', 'value': forum}],
+            'venue': venue, 'year': year, 'track': normalized_track,
+            'status': status, 'raw_decision': raw_status, 'evidence_url': evidence,
+            'observed_at': AS_OF, 'tags': tags,
+            'metadata': {**provenance, 'raw_track': track, 'raw_status': raw_status,
+                         'evidence_kind': 'curated_public_status_assertion',
+                         'original_decision_signature_verified': False,
+                         'source_snapshot_upper_bound': AS_OF}}, None
+
+
+def held_metadata(row, reason, provenance):
+    # Retractions and unresolved track scopes remain visible without becoming
+    # invented Reject/Withdrawn events in a schema that cannot represent them.
+    return {key: row.get(key) for key in ('id', 'title', 'abstract', 'track', 'status')} | {
+        'reason': reason, 'provenance': provenance}
+
+
+def collect(out, families=None, years=None, include_workshops=False, fetcher=None):
+    fetcher = fetcher or download
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    with (out / 'collect.lock').open('a') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        if (out / 'public-openreview-inventory.json').exists():
+            raise ValueError('Completed inventory exists; select a new output directory')
+        if shutil.disk_usage(out).free < 8 * 1024**3:
+            raise RuntimeError('Less than 8 GiB free before collection')
+        tree = json.loads(fetcher(TREE_URL, 8 * 1024**2))
+        if tree.get('truncated') is not False or tree.get('sha') != REVISION:
+            raise ValueError('Incomplete or unpinned public source tree')
+        selected = []
+        for item in tree['tree']:
+            match = re.fullmatch(r'venues/([a-z0-9]+)/\1((?:19|20)\d{2})\.jsonl', item['path'])
+            if item['type'] != 'blob' or not match:
+                continue
+            family, year = match[1], int(match[2])
+            if year > 2026 or (families is None and family == 'iclr'):
+                continue
+            if families is not None and family not in families or years is not None and year not in years:
+                continue
+            selected.append((item, family, year))
+        if not selected:
+            raise ValueError('No pinned source files match the request')
+        if families and set(families) - {family for _, family, _ in selected}:
+            raise ValueError('Requested venue has no selected source file')
+        if years and set(years) - {year for _, _, year in selected}:
+            raise ValueError('Requested year has no selected source file')
+        return _collect_files(out, selected, include_workshops, fetcher)
+
+
+def _collect_files(out, selected, include_workshops, fetcher):
+    summaries = []
+    groups = defaultdict(list)
+    global_forums = {}
+    with tempfile.TemporaryDirectory(prefix='.public-openreview-', dir=out) as temporary:
+        stage = Path(temporary)
+        held_path = stage / 'held-records.jsonl'
+        with held_path.open('w') as held:
+            for item, family, year in sorted(selected, key=lambda entry: entry[0]['path']):
+                size = item['size']
+                if not isinstance(size, int) or not 0 < size <= 96 * 1024**2:
+                    raise ValueError('Unexpected metadata file size')
+                data = fetcher(RAW_ROOT + item['path'], size)
+                blob = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
+                if len(data) != size or blob != item['sha']:
+                    raise ValueError('Pinned source blob checksum mismatch')
+                matrix = Counter()
+                reasons = Counter()
+                seen = set()
+                rows = 0
+                for number, line in enumerate(io.BytesIO(data), 1):
+                    if len(line) > 4 * 1024**2:
+                        raise ValueError('Oversized metadata row')
+                    row = json.loads(line)
+                    forum = row.get('id')
+                    if not isinstance(forum, str) or forum in seen:
+                        raise ValueError('Missing or duplicate source forum ID')
+                    seen.add(forum)
+                    edition = (family, year, row.get('track'))
+                    if forum in global_forums and global_forums[forum] != edition:
+                        raise ValueError('One forum appears in multiple source editions or tracks')
+                    global_forums[forum] = edition
+                    rows += 1
+                    matrix[(row.get('track'), row.get('status'))] += 1
+                    provenance = {'source_revision': REVISION, 'source_blob_sha1': blob,
+                                  'source_file': RAW_ROOT + item['path'], 'source_line': number}
+                    record, reason = normalize(row, family, year, provenance, include_workshops)
+                    if record is None:
+                        reasons[reason] += 1
+                        if reason != 'not_explicit_negative':
+                            held.write(json.dumps(held_metadata(row, reason, provenance), ensure_ascii=False) + '\n')
+                    else:
+                        groups[(family, year, record['track'])].append(record)
+                summary = {'path': item['path'], 'input_bytes': size, 'git_blob_sha1': blob,
+                           'rows': rows, 'excluded_or_held': dict(reasons),
+                           'status_matrix': [{'track': t, 'status': s, 'count': n} for (t, s), n in matrix.items()]}
+                summaries.append(summary)
+                print(json.dumps({key: summary[key] for key in ('path', 'rows', 'excluded_or_held')}), flush=True)
+        files = []
+        for (family, year, track), records in sorted(groups.items()):
+            name = f'public-openreview-{family}-{year}-{track}.jsonl'
+            path = stage / name
+            path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
+            manifest = {'source': SOURCE, 'parser_version': 1, 'venue': records[0]['venue'],
+                        'year': year, 'track': track, 'count': len(records), 'status': 'partial',
+                        'file': str(out / name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
+                        'list_url': REPOSITORY + '/tree/' + REVISION, 'source_revision': REVISION,
+                        'missing_abstract_count': sum(not row['abstract'] for row in records),
+                        'evidence_kind': 'curated_public_status_assertion',
+                        'note': 'Public subset only; no population coverage or original decision-signature claim.'}
+            path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
+            files.append(manifest)
+        result = {'source': SOURCE, 'revision': REVISION, 'finished_at': timestamp(),
+                  'input_rows': sum(item['rows'] for item in summaries), 'source_files': summaries,
+                  'normalized_records': sum(item['count'] for item in files), 'files': files,
+                  'include_workshops': include_workshops, 'held_file': str(out / held_path.name),
+                  'held_file_sha256': hashlib.sha256(held_path.read_bytes()).hexdigest(),
+                  'note': 'Zero explicit negative records in this mirror does not prove no public rejections exist.'}
+        inventory = stage / 'public-openreview-inventory.json'
+        inventory.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
+        # Publish only after every selected source has passed its pinned hash.
+        for path in sorted(stage.iterdir(), key=lambda path: path == inventory):
+            path.replace(out / path.name)
+        return result
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--out', required=True)
+    parser.add_argument('--family', action='append', help='Source family, e.g. icml, nips, emnlp')
+    parser.add_argument('--year', type=int, action='append')
+    parser.add_argument('--include-workshops', action='store_true')
+    args = parser.parse_args(argv)
+    collect(args.out, args.family, args.year, args.include_workshops)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/collectors/vision.py

@@ -5,10 +5,12 @@ import json
 from pathlib import Path
 import re
 import time
+from datetime import date
+from urllib.error import HTTPError
 from urllib.parse import parse_qs, unquote, urljoin, urlsplit
 
-from collect_pmlr import fetch
-from collect_proceedings import Document, Node, clean
+from paper_atlas.io.http import fetch
+from paper_atlas.io.html import Document, Node, clean
 
 ROOTS={'cvf':'https://openaccess.thecvf.com/','ecva':'https://www.ecva.net/papers.php'}
 HOSTS={'openaccess.thecvf.com','www.ecva.net','ecva.net'}
@@ -98,6 +100,14 @@ def discover(source,cache):
     return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track']))
 
 
+def author_names(raw):
+    raw=(raw or '').split(';',1)[0]
+    if ' and ' in raw and all(',' in name for name in raw.split(' and ')):
+        names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')]
+    else:names=raw.split(',')
+    return [clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()]
+
+
 def parse_paper(html,entry,source,venue,year,track,index_url):
     nodes=list(Document(html).root.walk());meta=defaultdict(list)
     for node in nodes:
@@ -112,11 +122,7 @@ def parse_paper(html,entry,source,venue,year,track,index_url):
         if clean(author):authors.append(clean(author))
     if not authors:
         raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or ''
-        raw=raw.split(';',1)[0]
-        if ' and ' in raw and all(',' in name for name in raw.split(' and ')):
-            names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')]
-        else:names=raw.split(',')
-        authors=[clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()]
+        authors=author_names(raw)
     dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)}
     if entry.get('doi'):dois.add(entry['doi'])
     if len(dois)>1:raise ValueError('Citation DOI differs from the official index')
@@ -126,45 +132,112 @@ def parse_paper(html,entry,source,venue,year,track,index_url):
             'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,
             'venue':venue,'year':year,'track':track,'status':'accepted',
             'raw_decision':'listed in official proceedings','list_url':index_url,
-            'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors')}}
+            'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors'),
+                        'index_urls':entry.get('index_urls',[index_url])}}
+
+
+def index_identity(url):
+    parsed=urlsplit(url)
+    match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path)
+    return match.groups() if parsed.netloc=='openaccess.thecvf.com' and match else None
+
+
+def read_index(index,cache):
+    url=index['index_url'];html=decode_html(fetch(url,cache,32*1024**2,delay=1))
+    links=set()
+    for node in Document(html).root.walk():
+        if node.tag!='a':continue
+        link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[])
+        if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue
+        if day[0]!='all':
+            try:valid=date.fromisoformat(day[0]).year==index['year']
+            except ValueError:valid=False
+            if not valid:continue
+        links.add(link)
+    all_pages=sorted(link for link in links if parse_qs(urlsplit(link).query)['day']==['all'])
+    day_pages=sorted(links-set(all_pages));index_errors=[]
+    if len(all_pages)>1:raise ValueError('Ambiguous all-papers index')
+    pages=[]
+    if all_pages:
+        try:pages=[(all_pages[0],decode_html(fetch(all_pages[0],cache,32*1024**2,delay=1)))]
+        except HTTPError as error:
+            if error.code!=404 or not day_pages:
+                error.close();raise
+            index_errors.append({'url':all_pages[0],'status':404});error.close()
+    if not pages and day_pages:
+        pages=[(page,decode_html(fetch(page,cache,32*1024**2,delay=1))) for page in day_pages]
+    if not pages:pages=[(url,html)]
+    entries={};raw_count=0
+    for page,document in pages:
+        parsed,count=index_entries(document,page);raw_count+=count
+        for entry in parsed:
+            entry['index_urls']=[page];old=entries.get(entry['url'])
+            if old:
+                for key in ('index_title','index_authors','doi'):
+                    if old.get(key) and entry.get(key) and old[key]!=entry[key]:raise ValueError('Conflicting metadata across official day indexes')
+                    if not old.get(key):old[key]=entry.get(key)
+                old['index_urls'].append(page)
+            else:entries[entry['url']]=entry
+    return list(entries.values()),raw_count,[page for page,_ in pages],index_errors
+
+
+def index_only_record(entry,source,venue,year,track,index_url,error):
+    if not entry['index_title'].strip():raise ValueError('Missing title in official index')
+    return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'],
+            'title':entry['index_title'],'abstract':None,'authors':author_names(entry.get('index_authors')),
+            'identifiers':{'doi':entry['doi']} if entry.get('doi') else {},'venue':venue,'year':year,'track':track,
+            'status':'accepted','raw_decision':'listed in official proceedings index',
+            'evidence_url':entry['index_urls'][0],'list_url':index_url,
+            'metadata':{'index_title':entry['index_title'],'index_authors':entry.get('index_authors'),
+                        'index_urls':entry['index_urls'],'abstract_fetch_error':{'status':error.code,'url':error.url,'message':str(error)}}}
 
 
 def collect_index(source,index,cache,out):
     url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track']
     entries=index.get('entries')
     raw_count=len(entries) if entries is not None else None
+    index_urls=[url];index_errors=[];missing_pages=[]
     if entries is None:
-        html=decode_html(fetch(url,cache,32*1024**2,delay=1))
-        # Some defaults contain only day one; prefer the explicitly linked all-papers page.
-        all_pages={urljoin(url,n.attrs.get('href','')) for n in Document(html).root.walk() if n.tag=='a' and parse_qs(urlsplit(urljoin(url,n.attrs.get('href',''))).query).get('day')==['all'] and urlsplit(urljoin(url,n.attrs.get('href',''))).netloc==urlsplit(url).netloc and urlsplit(urljoin(url,n.attrs.get('href',''))).path==urlsplit(url).path}
-        if len(all_pages)>1:raise ValueError('Ambiguous all-papers index')
-        if all_pages:
-            url=all_pages.pop();html=decode_html(fetch(url,cache,32*1024**2,delay=1))
-        entries,raw_count=index_entries(html,url)
+        entries,raw_count,index_urls,index_errors=read_index(index,cache)
+        if len(index_urls)==1:url=index_urls[0]
     records=[]
     for number,entry in enumerate(entries,1):
         if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition')
-        record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url)
+        try:record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url)
+        except HTTPError as error:
+            if error.code!=404:
+                error.close();raise
+            entry.setdefault('index_urls',[url])
+            record=index_only_record(entry,source,venue,year,track,url,error)
+            missing_pages.append(entry['url']);error.close()
         records.append(record)
         if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True)
     if not records:raise ValueError('Empty publication list')
     path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True)
     part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path)
     missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records)
-    manifest={'source':source,'parser_version':1,'venue':venue,'year':year,'track':track,
+    manifest={'source':source,'parser_version':2,'venue':venue,'year':year,'track':track,
               'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count,
               'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path),
-              'list_url':url,'status':'partial' if missing or missing_authors else 'collected'}
+              'index_urls':index_urls,'index_errors':index_errors,'missing_abstract_pages':missing_pages,
+              'list_url':url,'status':'partial' if missing or missing_authors or index_errors else 'collected'}
     path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True)
     return manifest
 
 
-def main():
-    parser=argparse.ArgumentParser();parser.add_argument('--source',choices=ROOTS,required=True)
+def main(argv=None, *, prog=None, source=None):
+    parser=argparse.ArgumentParser(prog=prog)
+    if source is None:parser.add_argument('--source',choices=ROOTS,required=True)
+    else:parser.set_defaults(source=source)
     parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True)
     parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append')
-    args=parser.parse_args();out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
+    parser.add_argument('--retry-catalogue',help='Select only blocked editions from an existing catalogue; write results to --out')
+    args=parser.parse_args(argv);out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
     indexes=discover(args.source,args.cache)
+    if args.retry_catalogue:
+        retry=json.loads(Path(args.retry_catalogue).read_text())
+        indexes=[index for index in indexes if retry.get(f'{index["venue"]}:{index["year"]}:{index["track"]}',{}).get('status')=='blocked']
+        if not indexes:parser.error('Retry catalogue has no blocked editions for this source')
     if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source')
     if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source')
     if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source')
@@ -179,7 +252,7 @@ def main():
         try:
             manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]}
         except Exception as exc:
-            catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc)}
+            catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc),'error_url':getattr(exc,'url',None)}
             print(json.dumps(catalogue[key]),flush=True)
         part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path)
     (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2))

paper_atlas/config.py

@@ -0,0 +1,4 @@
+"""Read-only resources shipped with the installed package."""
+from importlib.resources import files
+
+DEFAULT_REGISTRY = files('paper_atlas.resources').joinpath('venues.json')

paper_atlas/db/__init__.py

@@ -0,0 +1 @@
+"""Db."""

paper_atlas/db/backend.py

@@ -0,0 +1,50 @@
+"""Prevent legacy writers from diverging after an explicit PostgreSQL cutover."""
+from pathlib import Path
+
+
+def database_arguments(parser):
+    target = parser.add_mutually_exclusive_group(required=True)
+    target.add_argument('--db', help='Writable SQLite sample; never the frozen production backup')
+    target.add_argument('--pg-service', help='Protected libpq service for the published PostgreSQL dataset')
+    parser.add_argument('--pg-schema', help='Optional schema; otherwise use the configured search_path')
+
+
+def open_store(db=None, *, pg_service=None, pg_schema=None):
+    if bool(db) == bool(pg_service) or (pg_schema and not pg_service):
+        raise ValueError('Choose exactly one of SQLite db or PostgreSQL service; schema requires PostgreSQL')
+    if pg_service:
+        from paper_atlas.postgres.writer import PostgresStore
+        return PostgresStore(pg_service, pg_schema)
+    from paper_atlas.db.store import Store
+    return Store(db)
+
+
+def open_read_connection(db=None, *, pg_service=None, pg_schema=None):
+    if bool(db) == bool(pg_service) or (pg_schema and not pg_service):
+        raise ValueError('Choose exactly one of SQLite db or PostgreSQL service; schema requires PostgreSQL')
+    if pg_service:
+        from paper_atlas.postgres.client import ReadConnection
+        return ReadConnection(pg_service, pg_schema)
+    import sqlite3
+    path = Path(db).resolve()
+    if not path.is_file():
+        raise ValueError('Existing database required for read-only audit')
+    connection = sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)
+    try:
+        connection.execute('BEGIN')
+    except BaseException:
+        connection.close()
+        raise
+    return connection
+
+
+def inserted_id(connection, sql, params=()):
+    if getattr(connection, 'postgres', False):
+        return connection.execute(sql + ' RETURNING id', params).fetchone()[0]
+    return connection.execute(sql, params).lastrowid
+
+
+def require_sqlite_writable(path):
+    marker = Path(str(Path(path).resolve()) + '.postgresql-cutover.json')
+    if marker.exists():
+        raise RuntimeError('This SQLite source was cut over to PostgreSQL; use the configured PostgreSQL database. Marker: ' + str(marker))

paper_atlas/db/store.py

@@ -1,10 +1,12 @@
 """Shared identity resolution and provenance-preserving metadata storage."""
 import json
+from importlib.resources import files
 import re
 import sqlite3
 import unicodedata
 from pathlib import Path
-from submission_evidence import attach_submission,migrate_sources,write_decision
+from paper_atlas.db.submissions import attach_submission,migrate_sources,write_decision,latest_status
+from paper_atlas.db.backend import require_sqlite_writable, inserted_id
 
 
 def normalized(value):
@@ -28,14 +30,15 @@ def author_keys(authors):
 
 class Store:
     def __init__(self,path):
+        require_sqlite_writable(path)
         self.conn = sqlite3.connect(path,timeout=60)
-        self.conn.executescript(Path(__file__).with_name('schema.sql').read_text())
+        self.conn.executescript(files('paper_atlas.db').joinpath('schema.sql').read_text())
         self.conn.execute('PRAGMA synchronous=NORMAL')
         self.conn.execute('PRAGMA cache_size=-32768')
         migrate_sources(self.conn)
 
     def source(self,name,url):
-        self.conn.execute('INSERT OR IGNORE INTO sources(name,homepage) VALUES(?,?)',(name,url))
+        self.conn.execute('INSERT INTO sources(name,homepage) VALUES(?,?) ON CONFLICT DO NOTHING',(name,url))
         return self.conn.execute('SELECT id FROM sources WHERE name=?',(name,)).fetchone()[0]
 
     def merge(self,target,other,reason):
@@ -52,7 +55,7 @@ class Store:
         c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other))
         c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other))
         c.execute('UPDATE submissions SET paper_id=? WHERE paper_id=?',(target,other))
-        c.execute('INSERT OR IGNORE INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=?',(target,other))
+        c.execute('INSERT INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=? ON CONFLICT DO NOTHING',(target,other))
         c.execute('DELETE FROM paper_tags WHERE paper_id=?',(other,))
         c.execute('UPDATE paper_merges SET surviving_paper_id=? WHERE surviving_paper_id=?',(target,other))
         # Reattach pending candidate evidence without losing colliding rows.
@@ -62,7 +65,7 @@ class Store:
             left = target if left == other else left
             right = target if right == other else right
             if left != right:
-                c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(left,right),max(left,right),method,score,evidence,status))
+                c.execute('INSERT INTO match_candidates VALUES(?,?,?,?,?,?) ON CONFLICT DO NOTHING',(min(left,right),max(left,right),method,score,evidence,status))
         c.execute('DELETE FROM papers WHERE id=?',(other,))
 
     def record(self,source,external_id,url,title,abstract,authors,identifiers=None,metadata=None,published=None):
@@ -87,7 +90,7 @@ class Store:
             found = c.execute('SELECT paper_id FROM identifiers WHERE scheme=? AND value=?',(scheme,value)).fetchone()
             if found:
                 matches.add(found[0])
-        old = c.execute('SELECT paper_id,title,abstract FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
+        old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
         if old:
             matches.add(old[0])
         key = normalized(title)
@@ -104,18 +107,23 @@ class Store:
             for other in sorted(matches-{paper}):
                 self.merge(paper,other,'explicit identifier or exact title corroborated by authors')
             if old:
-                c.execute("UPDATE papers SET title=?,abstract=?,title_key=?,authors_json=? WHERE id=? AND title=? AND COALESCE(abstract,'')=COALESCE(?,'')",(title,abstract,key,json.dumps(authors,ensure_ascii=False),paper,old[1],old[2]))
+                old_abstract=old[2] or json.loads(old[4]).get('last_known_abstract',{}).get('abstract')
+                c.execute("UPDATE papers SET title=?,abstract=COALESCE(NULLIF(?,''),abstract),title_key=?,authors_json=CASE WHEN ?='[]' THEN authors_json ELSE ? END WHERE id=? AND title=? AND COALESCE(abstract,'')=COALESCE(?,'')",(title,abstract,key,json.dumps(authors,ensure_ascii=False),json.dumps(authors,ensure_ascii=False),paper,old[1],old_abstract))
             c.execute("UPDATE papers SET abstract=CASE WHEN abstract IS NULL OR abstract='' THEN ? ELSE abstract END WHERE id=?",(abstract,paper))
         else:
-            paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid
+            paper = inserted_id(c,'INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published))
         for scheme,value in ids:
-            c.execute('INSERT OR IGNORE INTO identifiers VALUES(?,?,?)',(scheme,value,paper))
+            c.execute('INSERT INTO identifiers VALUES(?,?,?) ON CONFLICT DO NOTHING',(scheme,value,paper))
         metadata=dict(metadata or {});metadata['record_identifiers']=dict(ids)
-        c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,url=excluded.url,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata,ensure_ascii=False,default=str)))
+        if old and not abstract:
+            previous={'abstract':old[2],'observed_at':old[3]} if old[2] else json.loads(old[4]).get('last_known_abstract')
+            if previous:metadata['last_known_abstract']=previous
+        from paper_atlas.db.submissions import timestamp
+        c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json,retrieved_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,url=excluded.url,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=excluded.retrieved_at',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata,ensure_ascii=False,default=str),timestamp()))
         record = c.execute('SELECT id FROM source_records WHERE source_id=? AND external_id=?',(source_id,external_id)).fetchone()[0]
         for other,_,_ in title_matches:
             if other != paper and c.execute('SELECT 1 FROM papers WHERE id=?',(other,)).fetchone():
-                c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending'))
+                c.execute('INSERT INTO match_candidates VALUES(?,?,?,?,?,?) ON CONFLICT DO NOTHING',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending'))
         return paper,record
 
     def record_tags(self,paper,record,source,tags):
@@ -130,23 +138,28 @@ class Store:
         c=self.conn
         c.execute('DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(record,evidence))
         for namespace,value in sorted(values):
-            c.execute('INSERT OR IGNORE INTO tags(namespace,value) VALUES(?,?)',(namespace,value))
+            c.execute('INSERT INTO tags(namespace,value) VALUES(?,?) ON CONFLICT DO NOTHING',(namespace,value))
             tag=c.execute('SELECT id FROM tags WHERE namespace=? AND value=?',(namespace,value)).fetchone()[0]
-            c.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,evidence))
+            c.execute('INSERT INTO paper_tags VALUES(?,?,?,?) ON CONFLICT DO NOTHING',(paper,tag,record,evidence))
 
-    def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None,identities=None,observed_at=None,decision_date=None,source_decision_id='',submitted_at=None):
+    def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None,identities=None,observed_at=None,decision_date=None,source_decision_id='',submitted_at=None,guard_status=False):
         c = self.conn
-        c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified'))
-        venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0]
+        existing=c.execute('SELECT id FROM venues WHERE lower(acronym)=lower(?)',(venue,)).fetchone()
+        if existing:venue_id=existing[0]
+        else:venue_id=inserted_id(c,"INSERT INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified'))
         for alias in aliases or []:
-            c.execute('INSERT OR IGNORE INTO venue_aliases VALUES(?,?)',(venue_id,alias))
-        c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track))
+            c.execute('INSERT INTO venue_aliases VALUES(?,?) ON CONFLICT DO NOTHING',(venue_id,alias))
+        c.execute('INSERT INTO editions(venue_id,year,track) VALUES(?,?,?) ON CONFLICT DO NOTHING',(venue_id,int(year),track))
         edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0]
         # A previous event in this row may already have merged the input paper.
         paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0]
         submission=attach_submission(self,edition,paper,record,external_id,identities or [])
+        if guard_status and status in ('rejected','desk_rejected','withdrawn'):
+            current = latest_status(c, submission)
+            if current and current not in ('unknown','under_review',status):
+                raise ValueError('Curated decision conflicts with current canonical submission status: ' + current)
         if submitted_at:
-            from submission_evidence import timestamp
+            from paper_atlas.db.submissions import timestamp
             c.execute('UPDATE submissions SET submitted_at=COALESCE(submitted_at,?) WHERE id=?',(timestamp(submitted_at),submission))
         if status is not None:write_decision(self,submission,record,status,raw_value,url,observed_at,decision_date,source_decision_id)
         return edition

paper_atlas/db/submissions.py

@@ -1,9 +1,10 @@
 """Canonical submission events with independent source and decision evidence."""
 from datetime import datetime, timezone
 import re
+from paper_atlas.io.publication_ids import mlsys_id
 
 
-PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','cvf','ecva','dblp'}
+PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp'}
 
 
 def timestamp(value=None):
@@ -15,11 +16,26 @@ def timestamp(value=None):
 def identities_for_row(row):
     identities=list(row.get('submission_identifiers') or [])
     own=row.get('identifiers') or {}
+    if row['source']=='mlsys':
+        value=own.get('publication_mlsys')
+        if (not isinstance(value,str) or not re.fullmatch(str(row['year'])+r'/[a-f0-9]{32}',value)
+                or row['venue']!='MLSys'):
+            raise ValueError('Invalid authoritative MLSys publisher identity')
     if own.get('openreview'):
         identities.append({'scheme':'openreview','value':own['openreview']})
     if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and own.get('doi'):
         doi=re.sub(r'^https?://(?:dx\.)?doi\.org/','',str(own['doi']),flags=re.I).strip().lower()
         if re.fullmatch(r'10\.\d{4,9}/\S+',doi):identities.append({'scheme':'publication_doi','value':doi})
+    publication_mlsys = own.get('publication_mlsys')
+    if not publication_mlsys and row['source']=='dblp' and row['venue']=='MLSys':
+        # Older normalized DBLP files retain their own publisher links even
+        # when a newly supported historical URL label was not yet parsed.
+        links = (row.get('metadata') or {}).get('bibliography', {}).get('ee', [])
+        candidates = {identity for link in links if (identity := mlsys_id(link))}
+        if len(candidates)==1:
+            publication_mlsys = candidates.pop()
+    if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and publication_mlsys:
+        identities.append({'scheme':'publication_mlsys','value':publication_mlsys})
     meta=row.get('metadata') or {}
     if row['source']=='iclr_program' and row['venue']=='ICLR' and meta.get('program_identity_type')=='submission' and meta.get('program_submission_number_verified') is True:
         identities.append({'scheme':'iclr_number','value':str(meta['program_source_id'])})
@@ -30,6 +46,9 @@ def identities_for_row(row):
             valid=re.fullmatch(r'[A-Za-z0-9_-]+',value) and not re.match(r'20\d{2}-Oral-',value)
         elif scheme=='iclr_number':valid=re.fullmatch(r'[1-9]\d*',value) and row['venue']=='ICLR'
         elif scheme=='publication_doi':valid=re.fullmatch(r'10\.\d{4,9}/\S+',value)
+        elif scheme=='publication_mlsys':
+            valid=(re.fullmatch(str(row['year'])+r'/[a-f0-9]{32}',value)
+                   and row['venue']=='MLSys' and row['source'] in PUBLICATIONS and row.get('status')=='accepted')
         else:valid=False
         if not valid:raise ValueError('Invalid authoritative submission identity')
         result.add((scheme,value))
@@ -39,21 +58,29 @@ def identities_for_row(row):
 def migrate_sources(c):
     if c.execute("SELECT 1 FROM schema_migrations WHERE name='submission_source_baseline_v1'").fetchone():return
     with c:
-        c.execute('INSERT OR IGNORE INTO submission_sources SELECT id,source_record_id,source_submission_id FROM submissions')
-        c.execute("INSERT OR IGNORE INTO decision_sources SELECT d.id,s.source_record_id,'' FROM decisions d JOIN submissions s ON s.id=d.submission_id")
+        c.execute('INSERT INTO submission_sources SELECT id,source_record_id,source_submission_id FROM submissions WHERE 1=1 ON CONFLICT DO NOTHING')
+        c.execute("INSERT INTO decision_sources SELECT d.id,s.source_record_id,'' FROM decisions d JOIN submissions s ON s.id=d.submission_id WHERE 1=1 ON CONFLICT DO NOTHING")
         c.execute("INSERT INTO schema_migrations(name) VALUES('submission_source_baseline_v1')")
 
 
 def next_id(c,table,audit,former):
     # Merged IDs are kept in audit; never allocate them to a later unrelated event.
+    if getattr(c, 'postgres', False):
+        return c.next_id(table)
     return max(c.execute('SELECT COALESCE(max(id),0) FROM '+table).fetchone()[0],c.execute('SELECT COALESCE(max('+former+'),0) FROM '+audit).fetchone()[0])+1
 
 
+def latest_status(c, submission):
+    row = c.execute('''SELECT status FROM decisions WHERE submission_id=?
+        ORDER BY observed_at DESC,COALESCE(decision_date,observed_at) DESC,id DESC LIMIT 1''', (submission,)).fetchone()
+    return row[0] if row else None
+
+
 def merge_submissions(c,target,other):
     if target==other:return
     editions=c.execute('SELECT id,edition_id FROM submissions WHERE id IN (?,?)',(target,other)).fetchall()
     if len(editions)!=2 or editions[0][1]!=editions[1][1]:raise ValueError('Cannot combine submissions from different editions')
-    c.execute('INSERT OR IGNORE INTO submission_sources SELECT ?,source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(target,other))
+    c.execute('INSERT INTO submission_sources SELECT ?,source_record_id,source_submission_id FROM submission_sources WHERE submission_id=? ON CONFLICT DO NOTHING',(target,other))
     c.execute('DELETE FROM submission_sources WHERE submission_id=?',(other,))
     for decision,status,raw,url,observed,date in c.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=?',(other,)).fetchall():
         duplicate=c.execute('SELECT id,decision_date FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND observed_at=?',(target,status,raw,url,observed)).fetchone()
@@ -61,7 +88,7 @@ def merge_submissions(c,target,other):
             surviving=duplicate[0]
             if date and duplicate[1] and date!=duplicate[1]:raise ValueError('Conflicting dates for the same decision observation')
             if date and not duplicate[1]:c.execute('UPDATE decisions SET decision_date=? WHERE id=?',(date,surviving))
-            c.execute('INSERT OR IGNORE INTO decision_sources SELECT ?,source_record_id,source_decision_id FROM decision_sources WHERE decision_id=?',(surviving,decision))
+            c.execute('INSERT INTO decision_sources SELECT ?,source_record_id,source_decision_id FROM decision_sources WHERE decision_id=? ON CONFLICT DO NOTHING',(surviving,decision))
             c.execute('DELETE FROM decision_sources WHERE decision_id=?',(decision,))
             c.execute('UPDATE decision_merges SET surviving_decision_id=? WHERE surviving_decision_id=?',(surviving,decision))
             c.execute("INSERT INTO decision_merges(former_decision_id,surviving_decision_id,reason) VALUES(?,?,'same decision observation on merged submission')",(decision,surviving))
@@ -95,9 +122,9 @@ def attach_submission(store,edition,paper,record,external_id,identities):
     else:
         submission=next_id(c,'submissions','submission_merges','former_submission_id')
         c.execute('INSERT INTO submissions(id,paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?,?)',(submission,paper,edition,record,external_id))
-    c.execute('INSERT OR IGNORE INTO submission_sources VALUES(?,?,?)',(submission,record,external_id))
+    c.execute('INSERT INTO submission_sources VALUES(?,?,?) ON CONFLICT DO NOTHING',(submission,record,external_id))
     for scheme,value in identities:
-        c.execute('INSERT OR IGNORE INTO submission_identifiers VALUES(?,?,?,?)',(edition,scheme,value,submission))
+        c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?) ON CONFLICT DO NOTHING',(edition,scheme,value,submission))
     return submission
 
 
@@ -111,5 +138,5 @@ def write_decision(store,submission,record,status,raw,url,observed_at=None,decis
     else:
         decision=next_id(c,'decisions','decision_merges','former_decision_id')
         c.execute('INSERT INTO decisions(id,submission_id,status,raw_value,evidence_url,observed_at,decision_date) VALUES(?,?,?,?,?,?,?)',(decision,submission,status,raw,url,observed,date))
-    c.execute('INSERT OR IGNORE INTO decision_sources VALUES(?,?,?)',(decision,record,source_decision_id or ''))
+    c.execute('INSERT INTO decision_sources VALUES(?,?,?) ON CONFLICT DO NOTHING',(decision,record,source_decision_id or ''))
     return decision

paper_atlas/ingestion/__init__.py

@@ -0,0 +1 @@
+"""Ingestion."""

paper_atlas/ingestion/arxiv.py

@@ -0,0 +1,89 @@
+"""Stream the official metadata ZIP into the configured metadata database."""
+import argparse
+import json
+import time
+import shutil
+import zipfile
+from pathlib import Path
+
+
+from paper_atlas.db.backend import database_arguments, open_store, inserted_id
+from paper_atlas.db.submissions import timestamp
+
+
+def refresh_categories(store,paper,record,categories):
+    """Keep category evidence for each arXiv record, including merged papers."""
+    if not isinstance(categories,str):raise ValueError('arXiv categories must be text')
+    store.record_tags(paper,record,'arxiv',[{'namespace':'arxiv:category','value':value} for value in categories.split()])
+    store.conn.execute("DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key='arxiv:categories' AND tag_id IN (SELECT id FROM tags WHERE namespace='arxiv:category')",(record,))
+
+
+def ingest(db, archive, *, pg_service=None, pg_schema=None, storage_path=None):
+    store = open_store(db,pg_service=pg_service,pg_schema=pg_schema)
+    conn = store.conn
+    run = None
+    try:
+        source = store.source('arxiv','https://arxiv.org')
+        previous = conn.execute('SELECT id,checkpoint_json FROM ingest_runs WHERE source_id=? AND status IN (\'running\',\'failed\') ORDER BY id DESC LIMIT 1', (source,)).fetchone()
+        fingerprint = {'path':str(Path(archive).resolve()), 'bytes':Path(archive).stat().st_size, 'mtime_ns':Path(archive).stat().st_mtime_ns}
+        start = 0
+        if previous:
+            checkpoint = json.loads(previous[1] or '{}')
+            if checkpoint.get('archive') == fingerprint:
+                run = previous[0]
+                start = checkpoint.get('line', 0)
+                conn.execute("UPDATE ingest_runs SET status='running',error=NULL WHERE id=?", (run,))
+            else:
+                raise RuntimeError('Pending run belongs to a different archive; resolve it explicitly')
+        else:
+            run = inserted_id(conn,'INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)', (source,json.dumps({'archive':fingerprint,'line':0})))
+        conn.commit()
+        line_number = start
+        last = time.monotonic()
+        monitored = Path(storage_path) if storage_path else (Path(db).parent if db else Path(archive).parent)
+        with zipfile.ZipFile(archive) as zipped:
+            # Reading the member to EOF also checks its CRC.
+            with zipped.open('arxiv-metadata-oai-snapshot.json') as stream:
+                for line_number, line in enumerate(stream, 1):
+                    if line_number <= start:
+                        continue
+                    row = json.loads(line)
+                    external_id = row['id']
+                    title = ' '.join(row['title'].split())
+                    abstract = row.get('abstract', '').strip()
+                    authors = row.get('authors_parsed') or row.get('authors') or []
+                    metadata = {k:v for k,v in row.items() if k not in ('title','abstract','authors','authors_parsed')}
+                    paper,record = store.record('arxiv',external_id,'https://arxiv.org/abs/'+external_id,title,abstract,authors,{'arxiv':external_id,'doi':row.get('doi')},metadata,(row.get('versions') or [{}])[0].get('created'))
+                    refresh_categories(store,paper,record,row.get('categories') or '')
+                    if line_number % 500 == 0:
+                        if shutil.disk_usage(monitored).free < 4*1024**3:
+                            raise RuntimeError('Stopped before exhausting disk: less than 4 GiB free')
+                        conn.execute('UPDATE ingest_runs SET records_seen=?,records_written=?,checkpoint_json=? WHERE id=?',(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
+                        conn.commit()
+                        if time.monotonic()-last >= 20:
+                            print(json.dumps({'run':run,'rows':line_number}),flush=True)
+                            last = time.monotonic()
+        conn.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
+        conn.commit()
+        print(json.dumps({'status':'complete','rows':line_number}),flush=True)
+    except BaseException as exc:
+        conn.rollback()
+        if run is not None:
+            conn.execute("UPDATE ingest_runs SET status='failed',error=? WHERE id=?",(str(exc),run))
+            conn.commit()
+        raise
+    finally:
+        conn.close()
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    database_arguments(parser)
+    parser.add_argument('--archive', required=True)
+    parser.add_argument('--storage-path', help='Local filesystem to monitor for free space; specify the database volume when running beside PostgreSQL')
+    args = parser.parse_args(argv)
+    ingest(args.db,args.archive,pg_service=args.pg_service,pg_schema=args.pg_schema,storage_path=args.storage_path)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/ingestion/arxiv_tags.py

@@ -0,0 +1,76 @@
+"""Rebuild per-record arXiv category evidence from retained source metadata."""
+import argparse
+from contextlib import closing
+import fcntl
+import json
+from pathlib import Path
+import shutil
+import sqlite3
+
+from paper_atlas.db.store import Store
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.ingestion.arxiv import refresh_categories
+
+
+def save(path,state):
+    part=path.with_suffix('.part');part.write_text(json.dumps(state,indent=2)+'\n');part.replace(path)
+
+
+def repair(db,out,resume=False,batch_size=1000):
+    db=Path(db).resolve();out=Path(out).resolve()
+    from paper_atlas.db.backend import require_sqlite_writable
+    require_sqlite_writable(db)
+    if not db.is_file():raise ValueError('Existing database required')
+    if batch_size<1 or batch_size>10000:raise ValueError('Batch size must be between 1 and 10000')
+    out.parent.mkdir(parents=True,exist_ok=True)
+    with Path(str(db)+'.maintenance.lock').open('a') as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        with closing(sqlite3.connect(db.as_uri()+'?mode=ro',uri=True)) as c:
+            source=c.execute("SELECT id FROM sources WHERE name='arxiv'").fetchone()
+            if not source:raise ValueError('No imported arXiv source')
+            total,maximum=c.execute('SELECT count(*),COALESCE(max(id),0) FROM source_records WHERE source_id=?',source).fetchone()
+        scope={'db':str(db),'source_id':source[0],'source_record_count':total,'maximum_source_record_id':maximum}
+        if resume:
+            state=json.loads(out.read_text())
+            if any(state.get(key)!=value for key,value in scope.items()):raise ValueError('Repair scope changed since checkpoint')
+            if state['status']=='complete':raise ValueError('Completed report exists; use a new report for a fresh repair')
+        else:
+            if out.exists():raise ValueError('Repair report exists; inspect it or explicitly resume')
+            state={**scope,'started_at':timestamp(),'last_record_id':0,'records_refreshed':0}
+        if shutil.disk_usage(db.parent).free<8*1024**3:raise RuntimeError('Less than 8 GiB free before repair')
+        state.update(status='running',error=None);save(out,state)
+        c=None
+        try:
+            store=Store(db);c=store.conn
+            while True:
+                rows=c.execute('SELECT id,paper_id,metadata_json FROM source_records WHERE source_id=? AND id>? AND id<=? ORDER BY id LIMIT ?',(source[0],state['last_record_id'],maximum,batch_size)).fetchall()
+                if not rows:break
+                for record,paper,payload in rows:
+                    categories=json.loads(payload).get('categories','')
+                    if categories is None:categories=''
+                    refresh_categories(store,paper,record,categories)
+                c.commit()
+                state.update(last_record_id=rows[-1][0],records_refreshed=state['records_refreshed']+len(rows));save(out,state)
+                if state['records_refreshed']%10000<len(rows):print(json.dumps(state),flush=True)
+                if shutil.disk_usage(db.parent).free<8*1024**3:raise RuntimeError('Less than 8 GiB free during repair')
+            if state['records_refreshed']!=total:raise ValueError('Repaired count differs from fixed source inventory')
+            current=c.execute('SELECT count(*),COALESCE(max(id),0) FROM source_records WHERE source_id=?',source).fetchone()
+            if current!=(total,maximum):raise ValueError('arXiv source inventory changed during repair')
+            legacy=c.execute("SELECT count(*) FROM paper_tags pt JOIN source_records r ON r.id=pt.source_record_id JOIN tags t ON t.id=pt.tag_id WHERE r.source_id=? AND t.namespace='arxiv:category' AND pt.evidence_key='arxiv:categories'",source).fetchone()[0]
+            if legacy:raise ValueError('Legacy category evidence remains after repair')
+            state.update(status='complete',finished_at=timestamp(),legacy_category_evidence_remaining=legacy);save(out,state)
+        except BaseException as error:
+            if c is not None:c.rollback()
+            state.update(status='failed',error=str(error));save(out,state);raise
+        finally:
+            if c is not None:c.close()
+    print(json.dumps(state),flush=True);return state
+
+
+def main(argv=None,*,prog=None):
+    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--db',required=True);parser.add_argument('--out',required=True)
+    parser.add_argument('--resume',action='store_true');parser.add_argument('--batch-size',type=int,default=1000);args=parser.parse_args(argv)
+    repair(args.db,args.out,args.resume,args.batch_size)
+
+
+if __name__=='__main__':main()

paper_atlas/ingestion/backfill.py

@@ -4,8 +4,9 @@ import hashlib
 import json
 from pathlib import Path
 
-from index_core import Store
-from submission_evidence import attach_submission,identities_for_row
+from paper_atlas.db.backend import database_arguments, open_store
+from paper_atlas.db.submissions import attach_submission,identities_for_row
+from paper_atlas.collectors.pmlr_tracks import require_track_manifest, require_track_record
 
 
 def queue_files(path):
@@ -18,14 +19,15 @@ def queue_files(path):
     return files
 
 
-def backfill(db,files):
+def backfill(db,files, *, pg_service=None, pg_schema=None):
     for filename in files:
         manifest=json.loads(Path(filename).with_suffix('.manifest.json').read_text())
+        require_track_manifest(manifest)
         if manifest['source'] in ('colm','neurips','iclr_program') and manifest.get('parser_version',0)<2:
             raise ValueError('Recollect old parser output before submission evidence backfill')
-    store=Store(db);c=store.conn;count=0
-    before=c.execute('SELECT count(*) FROM submissions').fetchone()[0]
+    store=open_store(db,pg_service=pg_service,pg_schema=pg_schema);c=store.conn;count=0
     try:
+        before=c.execute('SELECT count(*) FROM submissions').fetchone()[0]
         for filename in files:
             path=Path(filename);manifest=json.loads(path.with_suffix('.manifest.json').read_text());seen=set()
             with path.open() as stream:
@@ -35,8 +37,9 @@ def backfill(db,files):
                     seen.add(row['external_id'])
                     if any(row.get(key)!=manifest.get(key) for key in ('source','venue','year','track')):
                         raise ValueError('Evidence row differs from edition manifest')
+                    require_track_record(row, manifest)
                     record=c.execute('SELECT r.id,r.paper_id,r.metadata_json FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?',(row['source'],row['external_id'])).fetchone()
-                    edition=c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? AND e.year=? AND e.track=?',(row['venue'],row['year'],row['track'])).fetchone()
+                    edition=c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE lower(v.acronym)=lower(?) AND e.year=? AND e.track=?',(row['venue'],row['year'],row['track'])).fetchone()
                     if not record or not edition:raise ValueError('Evidence backfill requires an already imported record and edition')
                     if not c.execute('SELECT 1 FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id WHERE ss.source_record_id=? AND ss.source_submission_id=? AND s.edition_id=?',(record[0],row['external_id'],edition[0])).fetchone():
                         raise ValueError('No existing submission to backfill')
@@ -58,6 +61,10 @@ def backfill(db,files):
     finally:c.close()
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser();parser.add_argument('--db',required=True);parser.add_argument('--queue',required=True)
-    args=parser.parse_args();backfill(args.db,queue_files(args.queue))
+def main(argv=None, *, prog=None):
+    parser=argparse.ArgumentParser(prog=prog);database_arguments(parser);parser.add_argument('--queue',required=True)
+    args=parser.parse_args(argv);backfill(args.db,queue_files(args.queue),pg_service=args.pg_service,pg_schema=args.pg_schema)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/ingestion/maintenance.py

@@ -9,11 +9,13 @@ import json
 import fcntl
 import sqlite3
 import subprocess
+import sys
 from pathlib import Path
 
-from import_records import ingest_file
-from index_core import Store
-from backfill_submission_evidence import backfill,queue_files
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.backfill import backfill,queue_files
+from paper_atlas.db.backend import require_sqlite_writable
 
 
 UNIT='paper-atlas-arxiv-import.service'
@@ -24,6 +26,7 @@ def property_value(name):
 
 
 def merge_locked(db,archive,files,backfill_queue=None):
+    require_sqlite_writable(db)
     # Validate inputs before stopping the existing writer.
     for filename in files:
         path=Path(filename)
@@ -65,7 +68,7 @@ def merge_locked(db,archive,files,backfill_queue=None):
             if property_value('LoadState')=='loaded':
                 subprocess.run(['systemctl','start',UNIT],check=True)
             else:
-                subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).parent),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1','python3','-B',str(Path(__file__).with_name('import_arxiv.py')),'--db',db,'--archive',archive],check=True)
+                subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).resolve().parents[2]),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1',sys.executable,'-B','-m','paper_atlas','import','arxiv','--db',db,'--archive',archive],check=True)
             if property_value('ActiveState')!='active':
                 raise RuntimeError('arXiv unit did not resume; inspect the logs')
             print('ARXIV_RESUMED',flush=True)
@@ -78,14 +81,18 @@ def merge(db,archive,files,backfill_queue=None):
         return merge_locked(db,archive,files,backfill_queue)
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser()
+def main(argv=None, *, prog=None):
+    parser=argparse.ArgumentParser(prog=prog)
     parser.add_argument('--db',required=True)
     parser.add_argument('--archive',required=True)
     parser.add_argument('--schema-only',action='store_true',help='Apply idempotent schema/index updates at a saved checkpoint')
     parser.add_argument('--backfill-queue',help='Audited existing-source queue for canonical submission evidence')
     parser.add_argument('files',nargs='*')
-    args=parser.parse_args()
+    args=parser.parse_args(argv)
     if not args.files and not args.schema_only and not args.backfill_queue:
         parser.error('Provide collected files or --schema-only')
     merge(args.db,args.archive,args.files,args.backfill_queue)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/ingestion/pmlr_tracks.py

@@ -0,0 +1,203 @@
+"""Move existing PMLR submission events to verified tracks without reimporting."""
+import argparse
+from contextlib import closing
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+
+from paper_atlas.collectors.pmlr_tracks import require_track_manifest, require_track_record
+from paper_atlas.db.submissions import identities_for_row, timestamp
+
+
+def save(path, result):
+    part = path.with_suffix('.part')
+    part.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
+    part.replace(path)
+
+
+def load_inventories(paths):
+    scopes, seen, forums = [], set(), set()
+    for filename in paths:
+        path = Path(filename).resolve()
+        inventory = json.loads(path.read_text())
+        require_track_manifest(inventory)
+        if inventory.get('source') != 'pmlr' or inventory.get('venue') != 'ICML' or inventory.get('year', 0) < 2024:
+            raise ValueError('Only verified modern ICML PMLR volumes may be reclassified')
+        if not inventory.get('count') or not inventory.get('files'):
+            raise ValueError('A nonempty complete publication inventory is required')
+        rows, manifests = [], []
+        tracks = set()
+        for manifest in inventory['files']:
+            require_track_manifest(manifest)
+            if any(manifest.get(key) != inventory.get(key) for key in ('source', 'venue', 'year', 'volume', 'track_program')):
+                raise ValueError('Track manifest differs from volume inventory')
+            if manifest['track'] not in ('main', 'position') or manifest['track'] in tracks:
+                raise ValueError('Invalid or repeated track in volume inventory')
+            tracks.add(manifest['track'])
+            data = Path(manifest['file']).read_bytes()
+            if hashlib.sha256(data).hexdigest() != manifest.get('file_sha256'):
+                raise ValueError('Track file differs from its verified inventory')
+            chunk = [json.loads(line) for line in data.splitlines()]
+            if not chunk or len(chunk) != manifest['count']:
+                raise ValueError('Track inventory count mismatch')
+            for row in chunk:
+                if any(row.get(key) != manifest.get(key) for key in ('source', 'venue', 'year', 'track')):
+                    raise ValueError('Record differs from its track manifest')
+                if not row['external_id'].startswith('v' + str(inventory['volume']) + '/') or row['status'] != 'accepted':
+                    raise ValueError('Record is outside the accepted publication scope')
+                if row['external_id'] in seen:
+                    raise ValueError('Repeated publication in reclassification inventory')
+                seen.add(row['external_id'])
+                require_track_record(row, manifest)
+                own_forum = row.get('identifiers', {}).get('openreview')
+                if not own_forum or (row['year'], own_forum) in forums:
+                    raise ValueError('Missing or repeated forum identity in one ICML edition')
+                forums.add((row['year'], own_forum))
+            rows.extend(chunk); manifests.append(manifest)
+        if len(rows) != inventory['count']:
+            raise ValueError('Volume count differs from its track files')
+        scopes.append({'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
+                       'inventory': inventory, 'manifests': manifests, 'rows': rows})
+    if not scopes:
+        raise ValueError('At least one complete volume inventory is required')
+    return scopes
+
+
+def plan(c, scopes):
+    updates, changes = [], []
+    for scope in scopes:
+        volume = scope['inventory']['volume']
+        source_rows = c.execute('''SELECT r.id,r.external_id FROM source_records r JOIN sources src ON src.id=r.source_id
+            WHERE src.name='pmlr' AND r.external_id LIKE ?''', ('v' + str(volume) + '/%',)).fetchall()
+        if {r[1] for r in source_rows} != {r['external_id'] for r in scope['rows']}:
+            raise ValueError('Existing PMLR volume differs from the complete audited publication inventory')
+        for row in scope['rows']:
+            record = c.execute('''SELECT r.id,r.paper_id,r.title,r.abstract,r.authors_json,r.metadata_json FROM source_records r
+                JOIN sources src ON src.id=r.source_id WHERE src.name='pmlr' AND r.external_id=?''', (row['external_id'],)).fetchone()
+            if (record[2] != row['title'] or record[3] != row.get('abstract')
+                    or json.loads(record[4]) != row.get('authors', [])):
+                raise ValueError('Reclassification must preserve existing publisher text and authors: ' + row['external_id'])
+            metadata = json.loads(record[5])
+            forum = str(metadata.get('openreview') or (metadata.get('record_identifiers') or {}).get('openreview') or '')
+            forum = forum.split('id=')[-1].split('&')[0]
+            if not forum or forum != row['identifiers'].get('openreview'):
+                raise ValueError('Publisher forum identity differs from retained source metadata')
+            events = c.execute('''SELECT s.id,s.paper_id,s.edition_id,e.venue_id,e.year,e.track,v.acronym,s.source_record_id,s.source_submission_id
+                FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id JOIN editions e ON e.id=s.edition_id
+                JOIN venues v ON v.id=e.venue_id WHERE ss.source_record_id=? AND ss.source_submission_id=?''',
+                (record[0], row['external_id'])).fetchall()
+            if len(events) != 1:
+                raise ValueError('Publication must have exactly one existing submission event')
+            event = events[0]
+            sid, paper, old_edition, venue, year, old_track, acronym, owner, external = event
+            if paper != record[1] or year != row['year'] or acronym.upper() != 'ICML' or old_track not in ('main', 'position'):
+                raise ValueError('Existing submission is outside this publication scope')
+            identities = c.execute('SELECT scheme,value FROM submission_identifiers WHERE submission_id=? ORDER BY scheme,value', (sid,)).fetchall()
+            own = identities_for_row(row)
+            if {value for scheme, value in identities if scheme == 'openreview'} - {forum}:
+                raise ValueError('Existing event has a conflicting forum identity')
+            target = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?', (venue, year, row['track'])).fetchone()
+            if target:
+                for scheme, value in sorted(set(identities) | set(own)):
+                    collision = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target[0], scheme, value)).fetchone()
+                    if collision and collision[0] != sid:
+                        raise ValueError('Target track already has another event for this identity; reconcile sources before moving')
+                if c.execute('SELECT 1 FROM submissions WHERE source_record_id=? AND source_submission_id=? AND edition_id=? AND id<>?', (owner, external, target[0], sid)).fetchone():
+                    raise ValueError('Target track already has this publication event')
+            if old_track != row['track']:
+                links = c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?', (sid,)).fetchall()
+                if links != [(record[0], row['external_id'])] or owner != record[0] or external != row['external_id']:
+                    raise ValueError('Moving an event shared by multiple sources requires a separate reconciliation')
+                decisions = c.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY id', (sid,)).fetchall()
+                if not decisions or any(d[1] != 'accepted' for d in decisions):
+                    raise ValueError('Moving a publication with conflicting decision history requires reconciliation')
+                changes.append({'submission_id': sid, 'source_record_id': record[0], 'external_id': row['external_id'],
+                                'paper_id': paper, 'from_edition_id': old_edition, 'from_track': old_track,
+                                'to_track': row['track'], 'year': year, 'before_identifiers': identities,
+                                'before_decisions': decisions, 'track_evidence': row['metadata']['track_evidence']})
+            updates.append((row, record, event, own))
+    return updates, changes
+
+
+def reclassify(db, inventories, out, apply=False):
+    db, out = Path(db).resolve(), Path(out).resolve()
+    if apply:
+        from paper_atlas.db.backend import require_sqlite_writable
+        require_sqlite_writable(db)
+    if not db.is_file() or out == db:
+        raise ValueError('Existing database and distinct report path required')
+    scopes = load_inventories(inventories)
+    inputs = {Path(scope['path']) for scope in scopes}
+    inputs.update(Path(m['file']).resolve() for scope in scopes for m in scope['manifests'])
+    inputs.update(Path(m['file']).with_suffix('.manifest.json').resolve() for scope in scopes for m in scope['manifests'])
+    if out in inputs:
+        raise ValueError('Report must preserve evidence inputs')
+    out.parent.mkdir(parents=True, exist_ok=True)
+    with Path(str(db) + '.maintenance.lock').open('a') as lock, Path(str(out) + '.lock').open('a') as output_lock:
+        fcntl.flock(lock, (fcntl.LOCK_EX if apply else fcntl.LOCK_SH) | fcntl.LOCK_NB)
+        fcntl.flock(output_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        if out.exists():
+            raise ValueError('Reclassification report exists; use a new path')
+        mode = 'rw' if apply else 'ro'
+        with closing(sqlite3.connect(db.as_uri() + '?mode=' + mode, uri=True)) as c:
+            c.execute('PRAGMA foreign_keys=ON'); c.execute('PRAGMA cache_size=-32768')
+            c.execute('BEGIN IMMEDIATE' if apply else 'BEGIN')
+            updates, changes = plan(c, scopes)
+            result = {'status': 'planned', 'db': str(db), 'started_at': timestamp(),
+                      'records_verified': len(updates), 'events_to_move': len(changes), 'changes': changes,
+                      'inputs': [{key: scope[key] for key in ('path', 'sha256')} for scope in scopes],
+                      'note': 'Preserves paper/source/submission/decision IDs and all decision observations; no entity merges or text reimport.'}
+            save(out, result)
+            if not apply:
+                return result
+            try:
+                for row, record, event, own in updates:
+                    sid, _, old_edition, venue, year, old_track, _, _, _ = event
+                    c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)', (venue, year, row['track']))
+                    target = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?', (venue, year, row['track'])).fetchone()[0]
+                    if target != old_edition:
+                        c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (target, sid))
+                        c.execute('UPDATE submissions SET edition_id=? WHERE id=?', (target, sid))
+                    for scheme, value in own:
+                        c.execute('INSERT OR IGNORE INTO submission_identifiers VALUES(?,?,?,?)', (target, scheme, value, sid))
+                    metadata = json.loads(record[5]); metadata.update(row['metadata'])
+                    c.execute('UPDATE source_records SET metadata_json=? WHERE id=?', (json.dumps(metadata, ensure_ascii=False), record[0]))
+                for scope in scopes:
+                    for m in scope['manifests']:
+                        edition = c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? COLLATE NOCASE AND e.year=? AND e.track=?', ('ICML', m['year'], m['track'])).fetchone()[0]
+                        source = c.execute("SELECT id FROM sources WHERE name='pmlr'").fetchone()[0]
+                        c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at)
+                            VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
+                            expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL''',
+                            (edition, source, m['list_url'], 'partial' if m.get('missing_abstract_count') or m.get('status') == 'partial' else 'complete', m['count'], m['count'], timestamp()))
+                _, remaining = plan(c, scopes)
+                if remaining:
+                    raise ValueError('Post-migration track comparison still has mismatches')
+                print(json.dumps({'stage': 'checking_foreign_keys', 'events_moved': len(changes)}), flush=True)
+                if c.execute('PRAGMA foreign_key_check').fetchone():
+                    raise ValueError('Foreign key violations after track reclassification')
+                result.update(status='prepared', events_moved=len(changes))
+                save(out, result)
+                c.commit()
+            except BaseException as error:
+                c.rollback(); result.update(status='error', error=str(error)); save(out, result); raise
+            result.update(status='complete', finished_at=timestamp())
+            save(out, result)
+            return result
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--db', required=True)
+    parser.add_argument('--inventory', action='append', required=True)
+    parser.add_argument('--out', required=True)
+    parser.add_argument('--apply', action='store_true', help='Apply the verified migration under an exclusive maintenance lock')
+    args = parser.parse_args(argv)
+    result = reclassify(args.db, args.inventory, args.out, args.apply)
+    print(json.dumps({key: value for key, value in result.items() if key != 'changes'}), flush=True)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/ingestion/records.py

@@ -1,41 +1,50 @@
 """Import normalized conference JSONL files with coverage evidence."""
 import argparse
+import hashlib
 import json
 import time
 from pathlib import Path
 
-from index_core import Store
-from submission_evidence import identities_for_row,timestamp
+from paper_atlas.db.backend import database_arguments, open_store, inserted_id
+from paper_atlas.db.submissions import identities_for_row,timestamp
+from paper_atlas.collectors.pmlr_tracks import require_track_manifest, require_track_record
+from paper_atlas.collectors.public_openreview import SOURCE as PUBLIC_SOURCE
+from paper_atlas.collectors.icml_public_archive import SOURCE as ARCHIVE_SOURCE
 
 
-def ingest_file(db,path):
+def ingest_file(db,path, *, pg_service=None, pg_schema=None):
     path = Path(path)
     manifest_path = path.with_suffix('.manifest.json')
     manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
+    require_track_manifest(manifest)
     if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2:
         raise ValueError('Recollect proceedings with parser_version>=2 before identity matching')
     if manifest.get('source')=='iclr_program' and manifest.get('parser_version',0)<2:
         raise ValueError('Recollect ICLR with parser_version>=2 before submission identity matching')
-    store = Store(db)
-    c = store.conn
     source_name = manifest.get('source')
     if not source_name:
         raise ValueError('A source manifest is required')
-    source_id = store.source(source_name,manifest['list_url'])
-    run = c.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path)}))).lastrowid
-    c.commit()
+    store = open_store(db, pg_service=pg_service, pg_schema=pg_schema)
+    c = store.conn
+    run = None
     count = 0
     missing_abstracts = 0
     seen = set()
     editions = set()
     try:
-        with path.open() as stream:
+        source_id = store.source(source_name,manifest['list_url'])
+        run = inserted_id(c,'INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path),'expected_sha256':manifest.get('file_sha256')})))
+        c.commit()
+        digest = hashlib.sha256()
+        with path.open('rb') as stream:
             for line in stream:
+                digest.update(line)
                 row = json.loads(line)
                 if row['source'] != source_name:
                     raise ValueError('Source differs from manifest')
                 if any(key in manifest and manifest[key]!=row[key] for key in ('venue','year','track')):
                     raise ValueError('Conference edition differs from manifest')
+                require_track_record(row, manifest)
                 external = row['external_id']
                 if external in seen:
                     raise ValueError('Duplicate external ID in one collected file: '+external)
@@ -56,7 +65,7 @@ def ingest_file(db,path):
                         status=event['status']
                         if events and status is None:raise ValueError('Explicit decision event requires a status')
                         if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event')
-                        edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'))
+                        edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'),guard_status=source_name in (PUBLIC_SOURCE,ARCHIVE_SOURCE))
                     editions.add(edition)
                 else:
                     raise ValueError('This importer requires an explicit supported decision status')
@@ -64,6 +73,8 @@ def ingest_file(db,path):
                 missing_abstracts += not bool((row.get('abstract') or '').strip())
         if not count:
             raise ValueError('Empty metadata file')
+        if manifest.get('file_sha256') and digest.hexdigest() != manifest['file_sha256']:
+            raise ValueError('Collected file hash differs from manifest')
         if len(editions) != 1:
             raise ValueError('A manifest must describe exactly one conference edition and track')
         edition = next(iter(editions))
@@ -72,22 +83,27 @@ def ingest_file(db,path):
             raise ValueError(f'Manifest count {expected} differs from records {count}')
         complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0) and not missing_abstracts
         c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
-        c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run))
+        c.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),count,count,json.dumps({'file':str(path),'file_sha256':digest.hexdigest()}),run))
         c.commit()
         print(json.dumps({'file':str(path),'status':'imported','records':count,'missing_abstract_count':missing_abstracts}),flush=True)
     except BaseException as exc:
         c.rollback()
-        c.execute("UPDATE ingest_runs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(exc),run))
-        c.commit()
+        if run is not None:
+            c.execute("UPDATE ingest_runs SET status='failed',finished_at=?,records_seen=?,records_written=0,error=? WHERE id=?",(timestamp(),count,str(exc),run))
+            c.commit()
         raise
     finally:
         c.close()
 
 
-if __name__=='__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--db',required=True)
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    database_arguments(parser)
     parser.add_argument('files',nargs='+')
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     for filename in args.files:
-        ingest_file(args.db,filename)
+        ingest_file(args.db,filename,pg_service=args.pg_service,pg_schema=args.pg_schema)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/io/__init__.py

@@ -0,0 +1 @@
+"""Io."""

paper_atlas/io/html.py

@@ -0,0 +1,50 @@
+"""Small HTML tree used by official metadata adapters."""
+from html.parser import HTMLParser
+
+
+class Node:
+    def __init__(self,tag='',attrs=None):
+        self.tag=tag
+        self.attrs={key:value or '' for key,value in attrs or []}
+        self.children=[]
+
+    def text(self):
+        if self.tag in ('script','style'):
+            return ''
+        content=''.join(child.text() if isinstance(child,Node) else child for child in self.children)
+        return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content
+
+    def walk(self):
+        yield self
+        for child in self.children:
+            if isinstance(child,Node):
+                yield from child.walk()
+
+
+class Document(HTMLParser):
+    VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'}
+
+    def __init__(self,html):
+        super().__init__(convert_charrefs=True)
+        self.root=Node()
+        self.stack=[self.root]
+        self.feed(html)
+
+    def handle_starttag(self,tag,attrs):
+        node=Node(tag,attrs)
+        self.stack[-1].children.append(node)
+        if tag not in self.VOID:
+            self.stack.append(node)
+
+    def handle_endtag(self,tag):
+        for index in range(len(self.stack)-1,0,-1):
+            if self.stack[index].tag==tag:
+                del self.stack[index:]
+                break
+
+    def handle_data(self,data):
+        self.stack[-1].children.append(data)
+
+
+def clean(text):
+    return ' '.join(text.split())

paper_atlas/io/http.py

@@ -0,0 +1,50 @@
+"""Bounded public metadata requests and atomic response caching."""
+import hashlib
+from http.client import IncompleteRead
+import json
+from pathlib import Path
+import ssl
+import time
+import urllib.error
+import urllib.request
+
+
+def fetch(url,cache,limit=64*1024**2,delay=0.5):
+    cache = Path(cache)
+    cache.mkdir(parents=True,exist_ok=True)
+    path = cache/hashlib.sha256(url.encode()).hexdigest()
+    if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
+        if path.stat().st_size>limit:raise RuntimeError('Metadata response exceeds byte limit')
+        return path.read_bytes()
+    for attempt in range(3):
+        try:
+            request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
+            with urllib.request.urlopen(request,timeout=90) as response:
+                if response.status != 200:
+                    raise RuntimeError('Expected complete HTTP 200 response')
+                expected=int(response.headers.get('Content-Length','0'))
+                if expected>limit:
+                    raise RuntimeError('Metadata response exceeds byte limit')
+                data = response.read(limit+1)
+                if len(data)>limit:
+                    raise RuntimeError('Metadata response exceeds byte limit')
+                if expected and len(data)!=expected:
+                    raise IncompleteRead(data,max(0,expected-len(data)))
+            part = path.with_suffix('.part')
+            part.write_bytes(data)
+            part.replace(path)
+            path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
+            time.sleep(delay)
+            return data
+        except urllib.error.HTTPError as exc:
+            if exc.code not in (429,500,502,503,504) or attempt==2:
+                exc.close()
+                raise
+            retry_after=exc.headers.get('Retry-After','1');exc.close()
+            try:retry_after=int(retry_after)
+            except ValueError:retry_after=5
+            time.sleep(min(30,max(2**attempt,retry_after)))
+        except (urllib.error.URLError,TimeoutError,ConnectionError,IncompleteRead,ssl.SSLError) as exc:
+            reason=getattr(exc,'reason',exc)
+            if isinstance(reason,ssl.SSLCertVerificationError) or attempt==2:raise
+            time.sleep(min(30,2**(attempt+1)))

paper_atlas/io/publication_ids.py

@@ -0,0 +1,13 @@
+"""Stable identities from publisher-owned metadata self-links."""
+import re
+from urllib.parse import urlsplit
+
+
+def mlsys_id(url):
+    parsed = urlsplit(url)
+    if parsed.scheme not in ('https', 'http') or parsed.netloc != 'proceedings.mlsys.org':
+        return None
+    match = re.fullmatch(r'/paper(?:_files/paper)?/((?:19|20)\d{2})/hash/([a-f0-9]{32})-Abstract(?:-(?:Conference|mlsys(20\d{2})))?\.html', parsed.path)
+    if not match or match[3] and match[3] != match[1]:
+        return None
+    return match[1] + '/' + match[2]

paper_atlas/io/snapshots.py

@@ -98,9 +98,13 @@ def download(url,output,sha256=None,md5=None,expected_bytes=None,resume=False,mi
         return manifest
 
 
-if __name__=='__main__':
-    parser=argparse.ArgumentParser();parser.add_argument('--url',required=True);parser.add_argument('--output',required=True)
+def main(argv=None, *, prog=None):
+    parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--url',required=True);parser.add_argument('--output',required=True)
     parser.add_argument('--sha256');parser.add_argument('--md5');parser.add_argument('--expected-bytes',type=int)
     parser.add_argument('--resume',action='store_true');parser.add_argument('--min-free-gib',type=float,default=8)
-    args=parser.parse_args()
+    args=parser.parse_args(argv)
     print(json.dumps(download(args.url,args.output,args.sha256,args.md5,args.expected_bytes,args.resume,int(args.min_free_gib*1024**3))),flush=True)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/postgres/__init__.py

@@ -0,0 +1 @@
+"""PostgreSQL migration and read access for the paper index."""

paper_atlas/postgres/client.py

@@ -0,0 +1,96 @@
+"""Libpq service connections; credentials never belong in CLI arguments."""
+from collections.abc import Mapping
+import re
+from paper_atlas.postgres import text
+
+SEARCH_VECTOR = "to_tsvector('english'::regconfig,coalesce(title,'') || ' ' || coalesce(abstract,''))"
+
+
+def driver():
+    try:
+        import psycopg
+    except ImportError as error:
+        raise RuntimeError('Install paper-atlas[postgres] on the approved remote runner') from error
+    return psycopg
+
+
+def connect(service, *, readonly=False):
+    if not re.fullmatch(r'[A-Za-z0-9_-]+', service):
+        raise ValueError('Use a libpq service name, never a password or connection URL')
+    connection = driver().connect(service=service, connect_timeout=15, autocommit=True,
+                                  application_name='paper-atlas')
+    connection.execute("SET timezone='UTC'")
+    connection.execute("SET lock_timeout='5s'")
+    connection.execute("SET work_mem='2MB'")
+    connection.execute("SET maintenance_work_mem='16MB'")
+    connection.execute('SET max_parallel_workers_per_gather=0')
+    connection.execute('SET max_parallel_maintenance_workers=0')
+    if readonly:
+        connection.execute('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY')
+    return connection
+
+
+class Row(Mapping):
+    """The small mapping/positional interface used by existing query output."""
+    def __init__(self, columns, values):
+        self.columns, self.values = columns, values
+
+    def __getitem__(self, key):
+        return self.values[key if isinstance(key, int) else self.columns.index(key)]
+
+    def __iter__(self):
+        return iter(self.columns)
+
+    def __len__(self):
+        return len(self.columns)
+
+
+def row_factory(cursor, *, encoded=False):
+    columns = [column.name for column in cursor.description] if cursor.description else []
+    return lambda values: Row(columns, tuple(map(text.decode, values)) if encoded else values)
+
+
+class ReadConnection:
+    postgres = True
+
+    def __init__(self, service, schema=None):
+        if schema is not None and not re.fullmatch(r'[a-z][a-z0-9_]{0,35}', schema):
+            raise ValueError('Invalid schema name')
+        self.connection = connect(service, readonly=True)
+        from paper_atlas.postgres.schema import quote
+        if schema is not None:
+            self.connection.execute('SET LOCAL search_path TO ' + quote(schema) + ',pg_catalog')
+        if not self.has_table('papers'):
+            self.close()
+            raise RuntimeError('PostgreSQL publication is not ready in the configured search_path')
+        current = self.connection.execute('SELECT current_schema()').fetchone()[0]
+        # Schema comments are readable by other projects granted table access;
+        # consumers need no privileges on the private migration controller.
+        annotation = self.connection.execute("SELECT obj_description(oid,'pg_namespace') FROM pg_namespace WHERE nspname=%s", (current,)).fetchone()[0]
+        self.encoded = False
+        if annotation and annotation.startswith('paper-atlas:text-codec='):
+            codec = annotation.removeprefix('paper-atlas:text-codec=')
+            if codec != text.CODEC:
+                self.close()
+                raise RuntimeError('Unsupported PostgreSQL text storage codec')
+            self.encoded = codec == text.CODEC
+        self.connection.row_factory = lambda cursor: row_factory(cursor, encoded=self.encoded)
+
+    def execute(self, query, params=()):
+        # All caller SQL is static; user values are always bound parameters.
+        if 'sqlite_master' in query:
+            match = re.search(r"name='([a-z_]+)'", query)
+            if not match or match[1] not in ('decision_sources', 'submission_sources', 'venue_aliases'):
+                raise ValueError('Unsupported legacy table lookup')
+            return self.connection.execute('''SELECT 1 FROM information_schema.tables
+                WHERE table_schema=current_schema() AND table_name=%s''',
+                (match[1],))
+        values = tuple(map(text.encode, params)) if self.encoded else params
+        return self.connection.execute(query.replace('?', '%s'), values)
+
+    def has_table(self, name):
+        return bool(self.connection.execute('''SELECT 1 FROM information_schema.tables
+            WHERE table_schema=current_schema() AND table_name=%s''', (name,)).fetchone())
+
+    def close(self):
+        self.connection.close()

paper_atlas/postgres/migration.py

@@ -0,0 +1,279 @@
+"""Resumable, unpublished COPY followed by complete ordered content verification."""
+import argparse
+from contextlib import closing
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import re
+import shutil
+import sqlite3
+import uuid
+import zlib
+
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.postgres.client import connect, SEARCH_VECTOR
+from paper_atlas.postgres.schema import TABLES, inspect, fingerprint, quote, select_sql, table_sql
+from paper_atlas.postgres import text
+
+
+def digest_file(path):
+    digest = hashlib.sha256()
+    with path.open('rb') as stream:
+        while block := stream.read(8 * 1024 ** 2):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def update_hash(digest, row):
+    # Raw JSON columns remain text: whitespace, escapes and source evidence survive.
+    data = json.dumps(tuple(row), ensure_ascii=False, separators=(',', ':')).encode()
+    digest.update(len(data).to_bytes(8, 'big')); digest.update(data)
+
+
+def save(path, result):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    part = path.with_suffix(path.suffix + '.part')
+    part.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
+    part.replace(path)
+
+
+def free_space(path, minimum):
+    if path and shutil.disk_usage(path).free < minimum * 1024 ** 3:
+        raise RuntimeError('Migration storage headroom fell below the configured minimum')
+
+
+def qualify(schema, table):
+    return quote(schema) + '.' + quote(table)
+
+
+def add_constraints(pg, stage, plan):
+    def constraint(table, name, clause):
+        exists = pg.execute('''SELECT 1 FROM pg_constraint co JOIN pg_class cl ON cl.oid=co.conrelid
+            JOIN pg_namespace ns ON ns.oid=cl.relnamespace WHERE ns.nspname=%s AND cl.relname=%s AND co.conname=%s''',
+                            (stage, table, name)).fetchone()
+        if not exists:
+            pg.execute('ALTER TABLE ' + qualify(stage, table) + ' ADD CONSTRAINT ' + quote(name) + ' ' + clause)
+
+    for table, spec in plan.items():
+        constraint(table, table + '_pkey', 'PRIMARY KEY (' + ','.join(map(quote, spec['pk'])) + ')')
+        for number, index in enumerate(spec['indexes']):
+            if index['origin'] == 'u':
+                constraint(table, table + '_unique_' + str(number), 'UNIQUE (' + ','.join(quote(n) for n, desc in index['terms']) + ')')
+            else:
+                terms = ','.join(quote(n) + (' DESC' if desc else '') for n, desc in index['terms'])
+                pg.execute('CREATE ' + ('UNIQUE ' if index['unique'] else '') + 'INDEX IF NOT EXISTS ' +
+                           quote(index['name']) + ' ON ' + qualify(stage, table) + '(' + terms + ')')
+    for table, spec in plan.items():
+        for number, fk in enumerate(spec['foreign_keys']):
+            clause = ('FOREIGN KEY (' + ','.join(map(quote, fk['columns'])) + ') REFERENCES ' +
+                      qualify(stage, fk['table']) + '(' + ','.join(map(quote, fk['references'])) + ') NOT VALID')
+            constraint(table, table + '_fk_' + str(number), clause)
+        for number, check in enumerate(spec['checks']):
+            constraint(table, table + '_check_' + str(number), 'CHECK (' + check + ') NOT VALID')
+    constraints = pg.execute('''SELECT cl.relname,co.conname FROM pg_constraint co
+        JOIN pg_class cl ON cl.oid=co.conrelid JOIN pg_namespace ns ON ns.oid=cl.relnamespace
+        WHERE ns.nspname=%s AND NOT co.convalidated ORDER BY cl.relname,co.conname''', (stage,)).fetchall()
+    for table, name in constraints:
+        pg.execute('ALTER TABLE ' + qualify(stage, table) + ' VALIDATE CONSTRAINT ' + quote(name))
+
+
+def sequences(pg, source, stage, plan):
+    old = dict(source.execute('SELECT name,seq FROM sqlite_sequence'))
+    history = {'papers': ('paper_merges', 'former_paper_id'),
+               'submissions': ('submission_merges', 'former_submission_id'),
+               'decisions': ('decision_merges', 'former_decision_id')}
+    result = {}
+    for table, spec in plan.items():
+        if spec['pk'] != ['id']:
+            continue
+        value = max(source.execute('SELECT coalesce(max(id),0) FROM ' + quote(table)).fetchone()[0], old.get(table, 0))
+        if table in history:
+            audit, column = history[table]
+            value = max(value, source.execute('SELECT coalesce(max(' + quote(column) + '),0) FROM ' + quote(audit)).fetchone()[0])
+        sequence = pg.execute('SELECT pg_get_serial_sequence(%s,%s)', (qualify(stage, table), 'id')).fetchone()[0]
+        pg.execute('SELECT setval(%s,%s,%s)', (sequence, max(value, 1), value > 0))
+        result[table] = value
+    return result
+
+
+def upgrade_text_codec(pg, control, stage, plan):
+    """Upgrade committed legacy staging rows once; never recopy their source."""
+    changed = {}
+    with pg.transaction():
+        copied = {r[0] for r in pg.execute('SELECT name FROM ' + qualify(control, 'tables'))}
+        if not copied <= plan.keys():
+            raise ValueError('Unknown committed staging tables')
+        # An older run may have reached constraint validation. Replacing SOH
+        # can temporarily equal another legacy key even though the final mapping
+        # is bijective. Rebuild unpublished constraints/indexes after the upgrade.
+        constraints = pg.execute('''SELECT cl.relname,co.conname FROM pg_constraint co
+            JOIN pg_class cl ON cl.oid=co.conrelid JOIN pg_namespace ns ON ns.oid=cl.relnamespace
+            WHERE ns.nspname=%s AND co.contype IN ('f','p','u')
+            ORDER BY CASE WHEN co.contype='f' THEN 0 ELSE 1 END''', (stage,)).fetchall()
+        for table, name in constraints:
+            pg.execute('ALTER TABLE ' + qualify(stage, table) + ' DROP CONSTRAINT ' + quote(name))
+        indexes = pg.execute('''SELECT cl.relname FROM pg_index i JOIN pg_class cl ON cl.oid=i.indexrelid
+            JOIN pg_namespace ns ON ns.oid=cl.relnamespace WHERE ns.nspname=%s AND i.indisunique''', (stage,)).fetchall()
+        for name, in indexes:
+            pg.execute('DROP INDEX ' + qualify(stage, name))
+        for table in plan:
+            if table not in copied:
+                continue
+            columns = [quote(c['name']) for c in plan[table]['columns'] if c['type'] == 'TEXT']
+            if columns:
+                assignments = ','.join(c + '=replace(' + c + ',%s,%s)' for c in columns)
+                predicate = ' OR '.join('strpos(' + c + ',%s)>0' for c in columns)
+                params = [v for c in columns for v in (text.ESCAPE, text.encode(text.ESCAPE))]
+                params += [text.ESCAPE] * len(columns)
+                changed[table] = pg.execute('UPDATE ' + qualify(stage, table) + ' SET ' + assignments +
+                                            ' WHERE ' + predicate, params).rowcount
+        pg.execute('UPDATE ' + qualify(control, 'run') + ' SET text_codec=%s WHERE id=1', (text.CODEC,))
+    return changed
+
+
+def migrate(db, service, out, *, schema='research', storage_path=None, min_free_gib=8):
+    if not re.fullmatch(r'[a-z][a-z0-9_]{0,35}', schema):
+        raise ValueError('Use a short lowercase target schema name')
+    db, out = Path(db).resolve(), Path(out).resolve()
+    if not db.is_file() or out == db:
+        raise ValueError('Existing source and a distinct report are required')
+    if out.exists() and json.loads(out.read_text()).get('status') == 'complete':
+        raise ValueError('Completed report must be preserved')
+    free_space(storage_path, min_free_gib)
+    control = schema + '_migration'
+    report = {'status': 'preflight', 'source': str(db), 'service': service, 'schema': schema,
+              'started_at': timestamp(), 'tables': {}, 'scope': 'All 21 business tables; FTS5 internals rebuilt as native PostgreSQL GIN.'}
+    with Path(str(db) + '.maintenance.lock').open('a') as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        if Path(str(db) + '-wal').exists() and Path(str(db) + '-wal').stat().st_size:
+            raise ValueError('Freeze writers and checkpoint SQLite WAL before migrating')
+        with closing(sqlite3.connect(db.as_uri() + '?mode=ro', uri=True)) as source, closing(connect(service)) as pg:
+            source.execute('PRAGMA query_only=ON'); source.execute('PRAGMA cache_size=-16384'); source.execute('BEGIN')
+            plan = inspect(source)
+            stat = db.stat()
+            identity = {'file_sha256': digest_file(db), 'bytes': stat.st_size, 'mtime_ns': stat.st_mtime_ns,
+                        'schema_sha256': fingerprint(plan)}
+            report['source_identity'] = identity
+            advisory = zlib.crc32(schema.encode()) & 0x7fffffff
+            if not pg.execute('SELECT pg_try_advisory_lock(%s,%s)', (615173, advisory)).fetchone()[0]:
+                raise RuntimeError('Another migration holds the target lock')
+            if pg.execute('SELECT 1 FROM pg_namespace WHERE nspname=%s', (schema,)).fetchone():
+                raise ValueError('Target schema already exists; published data will never be overwritten')
+            pg.execute('CREATE SCHEMA IF NOT EXISTS ' + quote(control))
+            pg.execute('CREATE TABLE IF NOT EXISTS ' + qualify(control, 'run') +
+                       '(id INTEGER PRIMARY KEY CHECK(id=1), source_json TEXT NOT NULL, stage TEXT NOT NULL, status TEXT NOT NULL)')
+            pg.execute('CREATE TABLE IF NOT EXISTS ' + qualify(control, 'tables') +
+                       '(name TEXT PRIMARY KEY, rows BIGINT NOT NULL, sha256 TEXT NOT NULL, verified BOOLEAN NOT NULL DEFAULT false)')
+            pg.execute('ALTER TABLE ' + qualify(control, 'run') + " ADD COLUMN IF NOT EXISTS text_codec TEXT NOT NULL DEFAULT 'legacy'")
+            run = pg.execute('SELECT source_json,stage,status,text_codec FROM ' + qualify(control, 'run') + ' WHERE id=1').fetchone()
+            if run:
+                if json.loads(run[0]) != identity or run[2] == 'complete':
+                    raise ValueError('Resume requires the exact same unpublished SQLite snapshot')
+                stage = run[1]
+                if run[3] == 'legacy':
+                    report['legacy_text_upgrade_rows'] = upgrade_text_codec(pg, control, stage, plan)
+                elif run[3] != text.CODEC:
+                    raise ValueError('Unsupported staging text storage codec')
+            else:
+                stage = schema + '_stage_' + uuid.uuid4().hex[:12]
+                with pg.transaction():
+                    pg.execute('CREATE SCHEMA ' + quote(stage))
+                    for table, spec in plan.items():
+                        pg.execute(table_sql(stage, table, spec))
+                    pg.execute('INSERT INTO ' + qualify(control, 'run') +
+                               '(id,source_json,stage,status,text_codec) VALUES(1,%s,%s,%s,%s)',
+                               (json.dumps(identity, sort_keys=True), stage, 'copying', text.CODEC))
+            report.update(status='copying', stage=stage, text_codec=text.CODEC,
+                          content_hash_scope='Decoded original SQLite values; storage escapes are reversible.')
+            save(out, report)
+            try:
+                for table, spec in plan.items():
+                    free_space(storage_path, min_free_gib)
+                    previous = pg.execute('SELECT rows,sha256 FROM ' + qualify(control, 'tables') + ' WHERE name=%s', (table,)).fetchone()
+                    if previous:
+                        rows, digest = previous
+                    else:
+                        digest = hashlib.sha256(); rows = 0
+                        with pg.transaction():
+                            # Only this run's unpublished, incomplete table can be reset.
+                            pg.execute('TRUNCATE ' + qualify(stage, table))
+                            columns = ','.join(quote(c['name']) for c in spec['columns'])
+                            with pg.cursor().copy('COPY ' + qualify(stage, table) + '(' + columns + ') FROM STDIN') as copy:
+                                for row in source.execute(select_sql(None, table, spec)):
+                                    copy.write_row(tuple(map(text.encode, row))); update_hash(digest, row); rows += 1
+                                    if rows % 10000 == 0:
+                                        free_space(storage_path, min_free_gib)
+                                    if rows % 100000 == 0:
+                                        print(json.dumps({'stage': 'copying', 'table': table, 'rows': rows}), flush=True)
+                            digest = digest.hexdigest()
+                            pg.execute('INSERT INTO ' + qualify(control, 'tables') + '(name,rows,sha256) VALUES(%s,%s,%s)', (table, rows, digest))
+                    report['tables'][table] = {'source_rows': rows, 'source_sha256': digest}
+                    save(out, report)
+                    print(json.dumps({'stage': 'copied', 'table': table, 'rows': rows}), flush=True)
+                report['status'] = 'validating_constraints'; save(out, report)
+                add_constraints(pg, stage, plan)
+                report['sequences'] = sequences(pg, source, stage, plan)
+                report['status'] = 'verifying_contents'; save(out, report)
+                for table, spec in plan.items():
+                    digest = hashlib.sha256(); storage_digest = hashlib.sha256(); rows = 0
+                    with pg.transaction(), pg.cursor(name='verify_' + table) as cursor:
+                        cursor.itersize = 512
+                        cursor.execute(select_sql(stage, table, spec))
+                        for row in cursor:
+                            update_hash(storage_digest, row)
+                            update_hash(digest, tuple(map(text.decode, row))); rows += 1
+                    entry = report['tables'][table]
+                    entry.update(target_rows=rows, target_sha256=digest.hexdigest(),
+                                 target_storage_sha256=storage_digest.hexdigest())
+                    if rows != entry['source_rows'] or digest.hexdigest() != entry['source_sha256']:
+                        raise ValueError('Complete content verification failed for ' + table)
+                    pg.execute('UPDATE ' + qualify(control, 'tables') + ' SET verified=true WHERE name=%s', (table,))
+                    save(out, report)
+                    print(json.dumps({'stage': 'verified', 'table': table, 'rows': rows}), flush=True)
+                report['status'] = 'building_search_index'; save(out, report)
+                free_space(storage_path, min_free_gib)
+                pg.execute('CREATE INDEX IF NOT EXISTS papers_search_gin ON ' + qualify(stage, 'papers') + ' USING gin (' + SEARCH_VECTOR + ')')
+                search = pg.execute('''SELECT am.amname,i.indisvalid,i.indisready,t.relname
+                    FROM pg_index i JOIN pg_class ix ON ix.oid=i.indexrelid
+                    JOIN pg_class t ON t.oid=i.indrelid JOIN pg_namespace ns ON ns.oid=ix.relnamespace
+                    JOIN pg_am am ON am.oid=ix.relam WHERE ns.nspname=%s AND ix.relname='papers_search_gin' ''', (stage,)).fetchone()
+                if search != ('gin', True, True, 'papers'):
+                    raise ValueError('Native full-text index is not valid and ready')
+                for table in TABLES:
+                    pg.execute('ANALYZE ' + qualify(stage, table))
+                invalid = pg.execute('''SELECT count(*) FROM pg_constraint co JOIN pg_class cl ON cl.oid=co.conrelid
+                    JOIN pg_namespace ns ON ns.oid=cl.relnamespace WHERE ns.nspname=%s AND NOT co.convalidated''', (stage,)).fetchone()[0]
+                if invalid:
+                    raise ValueError('Unvalidated constraints remain')
+                if (db.stat().st_size, db.stat().st_mtime_ns) != (identity['bytes'], identity['mtime_ns']):
+                    raise ValueError('Source changed during migration')
+                if Path(str(db) + '-wal').exists() and Path(str(db) + '-wal').stat().st_size:
+                    raise ValueError('A source writer committed during migration')
+                report.update(status='prepared', foreign_keys='all_validated', checks='all_validated', native_search_index=True)
+                save(out, report)
+                with pg.transaction():
+                    pg.execute('COMMENT ON SCHEMA ' + quote(stage) + " IS 'paper-atlas:text-codec=" + text.CODEC + "'")
+                    pg.execute('ALTER SCHEMA ' + quote(stage) + ' RENAME TO ' + quote(schema))
+                    pg.execute('UPDATE ' + qualify(control, 'run') + " SET status='complete' WHERE id=1")
+                report.update(status='complete', finished_at=timestamp())
+                save(out, report)
+                return report
+            except BaseException as error:
+                report.update(status='error', error_type=type(error).__name__, error=str(error), failed_at=timestamp())
+                save(out, report)
+                raise
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('--db', required=True, help='Frozen SQLite source')
+    parser.add_argument('--pg-service', required=True, help='Protected libpq service name')
+    parser.add_argument('--schema', default='research')
+    parser.add_argument('--out', required=True)
+    parser.add_argument('--storage-path', help='Target filesystem path for free-space checks')
+    parser.add_argument('--min-free-gib', type=float, default=8)
+    args = parser.parse_args(argv)
+    result = migrate(args.db, args.pg_service, args.out, schema=args.schema,
+                     storage_path=args.storage_path, min_free_gib=args.min_free_gib)
+    print(json.dumps(result, ensure_ascii=False), flush=True)

paper_atlas/postgres/schema.py

@@ -0,0 +1,120 @@
+"""Inspect the complete business schema; rebuild SQLite engine internals natively."""
+import hashlib
+import json
+import re
+
+TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editions',
+          'venue_aliases', 'edition_sources', 'submissions', 'decisions',
+          'submission_sources', 'submission_identifiers', 'decision_sources',
+          'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
+          'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
+CHECKS = {
+    'venues': ["venue_type IN ('conference','journal','review_platform')"],
+    'edition_sources': ["status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')"],
+    'decisions': ["status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')"],
+    'match_candidates': ["status IN ('pending','merged','distinct')"],
+}
+TYPES = {'INTEGER': 'BIGINT', 'TEXT': 'TEXT COLLATE "C"', 'REAL': 'DOUBLE PRECISION'}
+
+
+def checks_in(ddl):
+    result = []
+    for match in re.finditer(r'\bCHECK\s*\(', ddl, re.I):
+        start = position = match.end()
+        depth, quoted = 1, False
+        while position < len(ddl) and depth:
+            char = ddl[position]
+            if char == "'":
+                if quoted and position + 1 < len(ddl) and ddl[position + 1] == "'":
+                    position += 2; continue
+                quoted = not quoted
+            elif not quoted:
+                depth += (char == '(') - (char == ')')
+            position += 1
+        if depth:
+            raise ValueError('Unclosed CHECK constraint')
+        result.append(ddl[start:position - 1])
+    return result
+
+
+def normalize_check(expression):
+    # Normalize SQL layout/keywords, retaining case and spaces inside literals.
+    tokens = re.findall(r"'(?:[^']|'')*'|[^']+", expression)
+    return ''.join(token if token.startswith("'") else re.sub(r'\s+', '', token).lower() for token in tokens)
+
+
+def quote(name):
+    return '"' + name.replace('"', '""') + '"'
+
+
+def inspect(connection):
+    business = {name for schema, name, kind, *_ in connection.execute('PRAGMA table_list')
+                if schema == 'main' and kind == 'table' and not name.startswith('sqlite_')}
+    if business != set(TABLES):
+        raise ValueError('Unexpected business tables: ' + repr(sorted(business ^ set(TABLES))))
+    result = {}
+    for table in TABLES:
+        columns = [{'name': r[1], 'type': r[2].upper(), 'notnull': bool(r[3]),
+                    'default': r[4], 'pk': r[5]}
+                   for r in connection.execute('PRAGMA table_info(' + quote(table) + ')')]
+        if any(c['type'] not in TYPES for c in columns):
+            raise ValueError('Unsupported storage type in ' + table)
+        pk = [c['name'] for c in sorted(columns, key=lambda c: c['pk']) if c['pk']]
+        if not pk:
+            raise ValueError('A deterministic primary key is required for ' + table)
+        indexes = []
+        for row in connection.execute('PRAGMA index_list(' + quote(table) + ')'):
+            if row[3] == 'pk':
+                continue
+            if row[4]:
+                raise ValueError('Partial indexes require explicit migration support')
+            terms = [(r[2], bool(r[3]), r[4]) for r in connection.execute('PRAGMA index_xinfo(' + quote(row[1]) + ')') if r[5]]
+            if any(name is None or coll != 'BINARY' for name, desc, coll in terms):
+                raise ValueError('Unsupported expression or collation in ' + table)
+            indexes.append({'name': row[1], 'unique': bool(row[2]), 'origin': row[3],
+                            'terms': [(name, desc) for name, desc, coll in terms]})
+        fks = {}
+        for row in connection.execute('PRAGMA foreign_key_list(' + quote(table) + ')'):
+            if row[5:7] != ('NO ACTION', 'NO ACTION'):
+                raise ValueError('Unsupported foreign key actions')
+            fk = fks.setdefault(row[0], {'table': row[2], 'columns': [], 'references': []})
+            fk['columns'].append(row[3]); fk['references'].append(row[4])
+        ddl = connection.execute("SELECT sql FROM sqlite_schema WHERE type='table' AND name=?", (table,)).fetchone()[0]
+        checks = CHECKS.get(table, [])
+        if sorted(map(normalize_check, checks_in(ddl))) != sorted(map(normalize_check, checks)):
+            raise ValueError('Unrecognized CHECK constraint in ' + table)
+        result[table] = {'columns': columns, 'pk': pk, 'indexes': indexes,
+                         'foreign_keys': list(fks.values()), 'checks': checks}
+    return result
+
+
+def fingerprint(plan):
+    return hashlib.sha256(json.dumps(plan, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
+
+
+def default_sql(value):
+    if value is None:
+        return ''
+    if value.upper() == 'CURRENT_TIMESTAMP':
+        return " DEFAULT to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+    if re.fullmatch(r'-?\d+|\'(?:[^\']|\'\')*\'', value):
+        return ' DEFAULT ' + value
+    raise ValueError('Unsupported column default')
+
+
+def table_sql(schema, table, spec):
+    columns = []
+    for column in spec['columns']:
+        generated = column['name'] == 'id' and spec['pk'] == ['id'] and column['type'] == 'INTEGER'
+        value = quote(column['name']) + ' ' + TYPES[column['type']]
+        value += ' GENERATED BY DEFAULT AS IDENTITY' if generated else default_sql(column['default'])
+        if column['notnull']:
+            value += ' NOT NULL'
+        columns.append(value)
+    return 'CREATE TABLE ' + quote(schema) + '.' + quote(table) + '(' + ','.join(columns) + ')'
+
+
+def select_sql(schema, table, spec):
+    prefix = quote(schema) + '.' if schema else ''
+    return ('SELECT ' + ','.join(quote(c['name']) for c in spec['columns']) +
+            ' FROM ' + prefix + quote(table) + ' ORDER BY ' + ','.join(quote(c) for c in spec['pk']))

paper_atlas/postgres/text.py

@@ -0,0 +1,31 @@
+"""Order-preserving, reversible storage for SQLite text containing NUL.
+
+PostgreSQL text forbids U+0000. Encode it as SOH/SOH, and encode a literal
+SOH as SOH/STX. All other characters stay unchanged. The mapping preserves
+binary lexicographic order, including string prefixes and composite keys.
+"""
+
+CODEC = 'nul-soh-v1'
+ESCAPE = '\x01'
+
+
+def encode(value):
+    if isinstance(value, str):
+        return value.replace(ESCAPE, ESCAPE + '\x02').replace('\x00', ESCAPE + ESCAPE)
+    return value
+
+
+def decode(value):
+    if not isinstance(value, str) or ESCAPE not in value:
+        return value
+    result = []
+    start = 0
+    while (position := value.find(ESCAPE, start)) >= 0:
+        result.append(value[start:position])
+        following = value[position + 1:position + 2]
+        if following not in (ESCAPE, '\x02'):
+            raise ValueError('Invalid nul-soh-v1 text escape')
+        result.append('\x00' if following == ESCAPE else ESCAPE)
+        start = position + 2
+    result.append(value[start:])
+    return ''.join(result)

paper_atlas/postgres/writer.py

@@ -0,0 +1,87 @@
+"""Native transactional writes reusing the shared identity/provenance policy."""
+import re
+import zlib
+
+from paper_atlas.db.store import Store
+from paper_atlas.postgres import text
+from paper_atlas.postgres.client import connect
+from paper_atlas.postgres.schema import TABLES, quote
+
+
+class Result:
+    def __init__(self, cursor):
+        self.cursor = cursor
+
+    def fetchone(self):
+        row = self.cursor.fetchone()
+        return tuple(map(text.decode, row)) if row is not None else None
+
+    def fetchall(self):
+        return [tuple(map(text.decode, row)) for row in self.cursor.fetchall()]
+
+    def __iter__(self):
+        return (tuple(map(text.decode, row)) for row in self.cursor)
+
+    @property
+    def rowcount(self):
+        return self.cursor.rowcount
+
+
+class WriteConnection:
+    postgres = True
+
+    def __init__(self, service, schema=None):
+        self.connection = connect(service)
+        try:
+            if schema is None:
+                schema = self.connection.execute('SELECT current_schema()').fetchone()[0]
+            if not isinstance(schema, str) or not re.fullmatch(r'[a-z][a-z0-9_]{0,35}', schema):
+                raise ValueError('Invalid PostgreSQL write schema')
+            annotation = self.connection.execute("SELECT obj_description(oid,'pg_namespace') FROM pg_namespace WHERE nspname=%s", (schema,)).fetchone()
+            if not annotation or annotation[0] != 'paper-atlas:text-codec=' + text.CODEC:
+                raise RuntimeError('Writes require a published dataset with the supported text codec')
+            tables = {r[0] for r in self.connection.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s AND table_type='BASE TABLE'", (schema,))}
+            if not set(TABLES) <= tables:
+                raise RuntimeError('Published business tables are incomplete or inaccessible')
+            if not self.connection.execute('SELECT pg_try_advisory_lock(%s,%s)', (615174, zlib.crc32(schema.encode()) & 0x7fffffff)).fetchone()[0]:
+                raise RuntimeError('Another writer holds this PostgreSQL dataset; retry after it closes')
+            self.schema = schema
+            self.connection.execute('SET search_path TO ' + quote(schema) + ',pg_catalog')
+            if not self.connection.execute("SELECT 1 FROM schema_migrations WHERE name='submission_source_baseline_v1'").fetchone():
+                raise RuntimeError('Submission provenance baseline is missing')
+            self.connection.autocommit = False
+        except BaseException:
+            self.connection.close()
+            raise
+
+    def execute(self, sql, params=()):
+        # The shared domain uses static SQL and bound parameters. It uses no
+        # SQLite-specific syntax; this adapter only binds/decodes text values.
+        return Result(self.connection.execute(sql.replace('?', '%s'), tuple(map(text.encode, params))))
+
+    def next_id(self, table):
+        if table not in ('submissions', 'decisions'):
+            raise ValueError('Unsupported explicit identity sequence')
+        return self.connection.execute("SELECT nextval(pg_get_serial_sequence(%s,'id'))", (quote(self.schema) + '.' + quote(table),)).fetchone()[0]
+
+    def commit(self):
+        self.connection.commit()
+
+    def rollback(self):
+        self.connection.rollback()
+
+    def close(self):
+        # Ends the session and releases its advisory lock, including after an
+        # error/rollback. File/chunk commits deliberately retain the writer lock.
+        self.connection.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, kind, error, traceback):
+        self.rollback() if kind else self.commit()
+
+
+class PostgresStore(Store):
+    def __init__(self, service, schema=None):
+        self.conn = WriteConnection(service, schema)

paper_atlas/querying.py

@@ -5,26 +5,56 @@ import sqlite3
 from pathlib import Path
 
 
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--db', required=True)
-    parser.add_argument('--query', help='SQLite FTS5 query over title and abstract')
+LATEST_DECISION = '''NOT EXISTS (
+    SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id
+    AND (newer.observed_at,COALESCE(newer.decision_date,newer.observed_at),newer.id)
+      > (d.observed_at,COALESCE(d.decision_date,d.observed_at),d.id)
+)'''
+ACCEPTED_PAPERS = "SELECT s.paper_id FROM submissions s JOIN decisions d ON d.submission_id=s.id WHERE d.status='accepted' AND "+LATEST_DECISION
+
+
+def known_acceptances(conn,paper):
+    return [dict(row) for row in conn.execute('''SELECT s.id AS submission_id,v.acronym AS venue,e.year,e.track,
+        d.id AS decision_id,d.decision_date,d.observed_at,d.evidence_url
+        FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
+        JOIN decisions d ON d.submission_id=s.id WHERE s.paper_id=? AND d.status='accepted' AND '''+LATEST_DECISION+
+        ' ORDER BY e.year,v.acronym,e.track,s.id',(paper,))]
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    database = parser.add_mutually_exclusive_group(required=True)
+    database.add_argument('--db', help='SQLite file')
+    database.add_argument('--pg-service', help='Protected libpq service; credentials stay outside CLI arguments')
+    parser.add_argument('--pg-schema', help='Optional PostgreSQL schema override; otherwise use the configured search_path')
+    parser.add_argument('--query', help='Title/abstract search: FTS5 for SQLite, web-search syntax for PostgreSQL')
     parser.add_argument('--paper',type=int,help='Paper ID; include sources, tags and submission history')
     parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI')
     parser.add_argument('--venue', help='Conference acronym')
     parser.add_argument('--year', type=int)
     parser.add_argument('--track',help='Exact track, e.g. main, findings, workshop')
     parser.add_argument('--status', choices=['accepted','rejected','desk_rejected','withdrawn','under_review','unknown'])
+    parser.add_argument('--outcome',choices=['accepted','no_known_acceptance'],help='跨会议的已知录用情况；未查到录用不等于从未录用')
     parser.add_argument('--limit', type=int, default=20)
     parser.add_argument('--stats', action='store_true')
-    args = parser.parse_args()
-    conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True)
-    conn.row_factory = sqlite3.Row
-    conn.execute('BEGIN')  # One consistent read snapshot while ingestion continues.
+    args = parser.parse_args(argv)
+    if args.pg_schema and not args.pg_service:
+        parser.error('--pg-schema requires --pg-service')
+    if args.pg_service:
+        from paper_atlas.postgres.client import ReadConnection
+        conn = ReadConnection(args.pg_service, schema=args.pg_schema) if args.pg_schema else ReadConnection(args.pg_service)
+    else:
+        conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True)
+        conn.row_factory = sqlite3.Row
+        conn.execute('BEGIN')  # One consistent read snapshot while ingestion continues.
     if args.stats:
         result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')}
         result['records_by_source'] = [dict(r) for r in conn.execute('SELECT s.name,count(r.id) AS imported_records FROM sources s LEFT JOIN source_records r ON r.source_id=s.id GROUP BY s.id ORDER BY s.name')]
         result['decision_events_by_status'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS events FROM decisions GROUP BY status')]
+        accepted=conn.execute('SELECT count(DISTINCT paper_id) FROM ('+ACCEPTED_PAPERS+')').fetchone()[0]
+        result['papers_with_known_acceptance']=accepted
+        result['papers_without_known_acceptance']=result['papers']-accepted
+        result['outcome_scope']='Only imported evidence is known. No known acceptance is not proof that a paper was never accepted.'
         result['coverage'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS sources FROM edition_sources GROUP BY status')]
         result['runs'] = [dict(r) for r in conn.execute('SELECT * FROM ingest_runs ORDER BY id DESC LIMIT 10')]
     else:
@@ -32,30 +62,30 @@ def main():
         if args.paper is not None:
             where.append('p.id=?'); params.append(args.paper)
         if args.query:
-            where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)'); params.append(args.query)
+            if args.pg_service:
+                from paper_atlas.postgres.client import SEARCH_VECTOR
+                where.append('p.id IN (SELECT id FROM papers WHERE '+SEARCH_VECTOR+" @@ websearch_to_tsquery('english'::regconfig,?))")
+            else:
+                where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)')
+            params.append(args.query)
         if args.tag:
             namespace,value = args.tag.split('=',1)
             where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value])
+        if args.outcome:
+            where.append('p.id '+('IN' if args.outcome=='accepted' else 'NOT IN')+' ('+ACCEPTED_PAPERS+')')
         if args.venue or args.year or args.status or args.track:
             clause = ['1=1']
             if args.venue:
                 if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='venue_aliases'").fetchone():
-                    clause.append('(v.acronym=? COLLATE NOCASE OR EXISTS(SELECT 1 FROM venue_aliases va WHERE va.venue_id=v.id AND va.alias=? COLLATE NOCASE))'); params.extend([args.venue,args.venue])
+                    clause.append('(lower(v.acronym)=lower(?) OR EXISTS(SELECT 1 FROM venue_aliases va WHERE va.venue_id=v.id AND lower(va.alias)=lower(?)))'); params.extend([args.venue,args.venue])
                 else:
-                    clause.append('v.acronym=? COLLATE NOCASE');params.append(args.venue)
+                    clause.append('lower(v.acronym)=lower(?)');params.append(args.venue)
             if args.track:
                 clause.append('e.track=?');params.append(args.track)
             if args.year:
                 clause.append('e.year=?'); params.append(args.year)
             if args.status:
-                latest='''EXISTS (
-                    SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=?
-                    AND NOT EXISTS (
-                        SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id
-                        AND (newer.observed_at,COALESCE(newer.decision_date,newer.observed_at),newer.id)
-                          > (d.observed_at,COALESCE(d.decision_date,d.observed_at),d.id)
-                    )
-                )'''
+                latest='EXISTS (SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? AND '+LATEST_DECISION+')'
                 clause.append('(NOT EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id) OR '+latest+')' if args.status=='unknown' else latest); params.append(args.status)
             where.append('p.id IN (SELECT s.paper_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')')
         sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?'
@@ -65,6 +95,8 @@ def main():
             paper = dict(row)
             paper['authors'] = json.loads(paper.pop('authors_json'))
             paper['identifiers'] = [dict(r) for r in conn.execute('SELECT scheme,value FROM identifiers WHERE paper_id=?',(row['id'],))]
+            paper['known_acceptances']=known_acceptances(conn,row['id'])
+            paper['publication_outcome']='accepted' if paper['known_acceptances'] else 'no_known_acceptance'
             if args.paper is not None:
                 paper['tags'] = [dict(r) for r in conn.execute('SELECT t.namespace,t.value,pt.evidence_key FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=?',(row['id'],))]
                 paper['sources'] = [dict(r) for r in conn.execute('SELECT s.name,r.external_id,r.url,r.title,r.abstract,r.retrieved_at FROM source_records r JOIN sources s ON s.id=r.source_id WHERE r.paper_id=?',(row['id'],))]
@@ -77,6 +109,7 @@ def main():
                         if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='decision_sources'").fetchone():
                             decision['sources']=[dict(r) for r in conn.execute('SELECT s.name,r.external_id,ds.source_decision_id FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?',(d['id'],))]
                         submission['decisions'].append(decision)
+                    submission['latest_status']=submission['decisions'][-1]['status'] if submission['decisions'] else 'unknown'
                     if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='submission_sources'").fetchone():
                         submission['sources']=[dict(r) for r in conn.execute('SELECT so.name,r.external_id,ss.source_submission_id FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id JOIN sources so ON so.id=r.source_id WHERE ss.submission_id=?',(sub['id'],))]
                     submissions.append(submission)

paper_atlas/resources/__init__.py

@@ -0,0 +1 @@
+"""Resources."""

paper_atlas/resources/venues.json

@@ -12,7 +12,9 @@
     "ijcai": "https://www.ijcai.org/all_proceedings/",
     "dblp": "https://dblp.org/db/conf/",
     "colm": "https://proceedings.colm.cc/",
-    "iclr_program": "https://iclr.cc/virtual/2026/papers.html"
+    "iclr_program": "https://iclr.cc/virtual/2026/papers.html",
+    "mlsys": "https://proceedings.mlsys.org/",
+    "acmmm": "https://www.acmmm.org/"
   },
   "venues": [
     {
@@ -643,6 +645,37 @@
         "main"
       ],
       "coverage": "pending"
+    },
+    {
+      "acronym": "MLSys",
+      "field": "ml_systems",
+      "venue_type": "conference",
+      "discovery_sources": [
+        "mlsys",
+        "dblp"
+      ],
+      "scope": "user_confirmed_related",
+      "years": "all_publicly_available",
+      "tracks": [
+        "main"
+      ],
+      "coverage": "pending"
+    },
+    {
+      "acronym": "MM",
+      "field": "multimodal",
+      "venue_type": "conference",
+      "discovery_sources": [
+        "dblp",
+        "acmmm",
+        "openreview"
+      ],
+      "scope": "user_confirmed_related",
+      "years": "all_publicly_available",
+      "tracks": [
+        "main"
+      ],
+      "coverage": "pending"
     }
   ],
   "additional_tracks": [

pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "paper-atlas"
+version = "0.1.0"
+description = "Long-term, on-demand paper metadata and public submission history index"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = ["PyYAML==6.0.3"]
+
+[project.optional-dependencies]
+postgres = ["psycopg[binary]==3.3.8"]
+
+[project.scripts]
+paper-atlas = "paper_atlas.cli:main"
+
+[tool.setuptools.packages.find]
+include = ["paper_atlas*"]
+exclude = ["tests*"]
+
+[tool.setuptools.package-data]
+"paper_atlas.db" = ["schema.sql"]
+"paper_atlas.resources" = ["venues.json"]

requirements.txt

@@ -1 +0,0 @@
-PyYAML==6.0.3

test_query.py

@@ -1,51 +0,0 @@
-import json
-import subprocess
-import sys
-import tempfile
-import unittest
-from pathlib import Path
-
-from index_core import Store
-
-
-class QueryTests(unittest.TestCase):
-    def test_joint_conference_alias_does_not_duplicate_submission(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            db=Path(tmp)/'papers.sqlite';store=Store(db);self.addCleanup(store.conn.close)
-            paper,record=store.record('acl_anthology','paper1','https://example.org/1','Joint event paper','Abstract',[])
-            store.decision(paper,record,'COLING+LREC',2024,'main','paper1','https://example.org/1','accepted','official XML',['COLING','LREC'])
-            store.conn.commit()
-            for venue in ('COLING','LREC'):
-                command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--venue',venue,'--track','main']
-                rows=json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout)
-                self.assertEqual([r['id'] for r in rows],[paper])
-            self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
-
-    def test_venue_filter_is_case_insensitive_and_uses_latest_decision(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            db=Path(tmp)/'papers.sqlite'
-            store=Store(db)
-            self.addCleanup(store.conn.close)
-            paper,record=store.record('peerread','paper1','https://example.org/paper1','A real submission','Abstract',['Jane Doe'])
-            store.decision(paper,record,'ICLR',2017,'main','paper1','https://example.org/paper1','rejected','explicit rejection')
-            store.record('arxiv','2601.00001','https://example.org/other','Another paper','Abstract',['John Smith'])
-            store.conn.commit()
-            command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--venue','iclr','--year','2017','--status','rejected']
-            result=json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout)
-            self.assertEqual([r['id'] for r in result],[paper])
-            details_command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--paper',str(paper)]
-            details=json.loads(subprocess.run(details_command,check=True,capture_output=True,text=True).stdout)[0]
-            self.assertEqual(details['authors'],['Jane Doe'])
-            self.assertEqual(details['submissions'][0]['decisions'][0]['status'],'rejected')
-            self.assertEqual(details['sources'][0]['external_id'],'paper1')
-            stats_command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--stats']
-            stats=json.loads(subprocess.run(stats_command,check=True,capture_output=True,text=True).stdout)
-            self.assertEqual(stats['papers'],2)
-            self.assertEqual({r['name']:r['imported_records'] for r in stats['records_by_source']},{'arxiv':1,'peerread':1})
-            store.decision(paper,record,'ICLR',2017,'main','paper1','https://example.org/paper1','accepted','changed decision')
-            store.conn.commit()
-            self.assertEqual(json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout),[])
-
-
-if __name__=='__main__':
-    unittest.main()

test_vision.py

@@ -1,72 +0,0 @@
-import tempfile
-import unittest
-from unittest.mock import patch
-
-from collect_vision import collect_index, decode_html, discover, index_entries, parse_paper
-
-
-class VisionTests(unittest.TestCase):
-    def entry(self):
-        return {'url':'https://www.ecva.net/papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php','index_title':'Paper','index_authors':'Jane Doe*, John Smith','doi':'10.1007/example_1'}
-
-    def test_ecva_doi_belongs_to_its_own_title_entry(self):
-        html='<dl><dt class="ptitle"><a href="papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php">First paper</a></dt><dd>Jane Doe*, John Smith</dd><dd><a href="https://link.springer.com/chapter/10.1007/example_1">DOI</a></dd><dt class="ptitle"><a href="papers/eccv_2024/papers_ECCV/html/2_ECCV_2024_paper.php">Second paper</a></dt><dd>Other Author</dd><dd><a href="https://doi.org/10.1007/example_2">DOI</a></dd></dl>'
-        entries,count=index_entries(html,'https://www.ecva.net/papers.php')
-        self.assertEqual(count,2)
-        self.assertEqual([e['doi'] for e in entries],['10.1007/example_1','10.1007/example_2'])
-        html='<div id="papertitle">Paper</div><div id="abstract">Abstract citing <a href="https://arxiv.org/abs/1706.03762">another paper</a> and <a href="https://doi.org/10.1007/wrong">a reference</a>.</div>'
-        row=parse_paper(html,entries[0],'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php')
-        self.assertEqual(row['identifiers'],{'doi':'10.1007/example_1'})
-        self.assertEqual(row['authors'],['Jane Doe','John Smith'])
-
-    def test_cvf_meta_authors_and_missing_abstract_never_capture_footer(self):
-        html='<meta name="citation_title" content="A vision paper"><meta name="citation_author" content="Doe, Jane"><div id="authors">Do not include the proceedings citation</div><p>Copyright footer</p>'
-        entry=dict(self.entry(),doi=None)
-        row=parse_paper(html,entry,'cvf','CVPR',2013,'main','https://openaccess.thecvf.com/CVPR2013')
-        self.assertEqual(row['authors'],['Jane Doe'])
-        self.assertIsNone(row['abstract'])
-        self.assertEqual(row['title'],'A vision paper')
-
-    def test_conflicting_self_doi_is_rejected(self):
-        html='<meta name="citation_title" content="Paper"><meta name="citation_doi" content="10.1007/different">'
-        with self.assertRaisesRegex(ValueError,'DOI differs'):
-            parse_paper(html,self.entry(),'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php')
-
-    def test_legacy_ecva_author_formats_do_not_become_extra_people(self):
-        entry=dict(self.entry(),index_authors='Doe, Jane and Smith, John')
-        page='<div id="papertitle">Paper</div><div id="authors">Jane Doe, John Smith; The European Conference on Computer Vision (ECCV), 2018, pp. 1-12</div>'
-        row=parse_paper(page,entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php')
-        self.assertEqual(row['authors'],['Jane Doe','John Smith'])
-        row=parse_paper('<div id="papertitle">Paper</div>',entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php')
-        self.assertEqual(row['authors'],['Jane Doe','John Smith'])
-
-    def test_cvpr_day_one_is_replaced_by_explicit_all_papers_index(self):
-        root='https://openaccess.thecvf.com/CVPR2026';all_url=root+'?day=all'
-        first='https://openaccess.thecvf.com/content/CVPR2026/html/First_CVPR_2026_paper.html'
-        second='https://openaccess.thecvf.com/content/CVPR2026/html/Second_CVPR_2026_paper.html'
-        def item(url):return f'<dt class="ptitle"><a href="{url}">Paper</a></dt><dd>Jane Doe, John Smith</dd><dd>[<a href="never-download.pdf">pdf</a>]</dd>'
-        pages={root:f'<a href="?day=all">All papers</a><dl>{item(first)}</dl>',all_url:f'<dl>{item(first)}{item(second)}</dl>',first:'<div id="papertitle">First paper</div><div id="abstract">First abstract</div>',second:'<div id="papertitle">Second paper</div><p>Footer without an abstract</p>'}
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
-            manifest=collect_index('cvf',{'venue':'CVPR','year':2026,'track':'main','index_url':root},tmp,tmp)
-        self.assertEqual(manifest['count'],2)
-        self.assertEqual(manifest['list_url'],all_url)
-        self.assertEqual(manifest['status'],'partial')
-        self.assertEqual(manifest['missing_abstract_count'],1)
-        self.assertEqual({call.args[0] for call in fetch.call_args_list},{root,all_url,first,second})
-
-    def test_discovery_keeps_findings_separate_and_excludes_workshops(self):
-        html='<a href="/CVPR2026">Main</a><a href="/CVPR2026_findings">Findings</a><a href="/ICCV2025">Main</a><a href="/CVPR2013.py">Legacy main</a><a href="/CVPR2026_workshops/menu">Workshops</a><a href="/WACV2026">WACV</a>'
-        with patch('collect_vision.fetch',return_value=html.encode()):indexes=discover('cvf','unused')
-        self.assertEqual({(x['venue'],x['year'],x['track']) for x in indexes},{('CVPR',2013,'main'),('CVPR',2026,'main'),('CVPR',2026,'findings'),('ICCV',2025,'main')})
-
-    def test_unparsed_publication_link_does_not_silently_reduce_coverage(self):
-        html='<a href="content/CVPR2026/html/Unparsed_CVPR_2026_paper.html">Other paper</a><dl><dt><a href="content/CVPR2026/html/First_CVPR_2026_paper.html">Paper</a></dt></dl>'
-        with self.assertRaisesRegex(ValueError,'outside parsed'):
-            index_entries(html,'https://openaccess.thecvf.com/')
-
-    def test_legacy_declared_encoding_is_respected(self):
-        html='<meta content="text/html; charset=ISO-8859-1"><div>André</div>'
-        self.assertIn('André',decode_html(html.encode('iso-8859-1')))
-
-
-if __name__=='__main__':unittest.main()

tests/__init__.py

@@ -0,0 +1 @@
+"""Tests."""

tests/test_acl.py

@@ -4,7 +4,7 @@ import unittest
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
-from collect_acl import paper_id,volume_scope,collect_xml
+from paper_atlas.collectors.acl import paper_id,volume_scope,collect_xml
 
 
 class AnthologyTests(unittest.TestCase):

tests/test_arxiv_audit.py

@@ -0,0 +1,44 @@
+from types import SimpleNamespace
+from unittest.mock import patch
+import json
+from pathlib import Path
+import sqlite3
+from contextlib import closing
+import tempfile
+import unittest
+import zipfile
+
+from paper_atlas.audits.arxiv import audit,MEMBER
+
+
+class ArxivSourceAuditTests(unittest.TestCase):
+    def setUp(self):
+        self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup);self.root=Path(self.temp.name)
+        self.archive=self.root/'metadata.zip';self.out=self.root/'audit.json'
+        disk=patch('paper_atlas.audits.arxiv.shutil.disk_usage',return_value=SimpleNamespace(free=16*1024**3));disk.start();self.addCleanup(disk.stop)
+
+    def write(self,body):
+        with zipfile.ZipFile(self.archive,'w',compression=zipfile.ZIP_DEFLATED) as z:z.writestr(MEMBER,body)
+
+    def test_repeated_ids_and_payload_changes_are_distinct_from_unique_papers(self):
+        one={'id':'2601.00001','title':'A paper','abstract':'First text'};updated={**one,'abstract':'Revised text'}
+        two={'id':'2601.00002','title':'Another paper','abstract':''}
+        self.write('\n'.join(json.dumps(row) for row in (one,updated,updated,two)))
+        result=audit(self.archive,self.out)
+        self.assertEqual((result['raw_rows'],result['unique_external_ids'],result['extra_rows_for_repeated_ids']),(4,2,2))
+        self.assertTrue(result['zip_eof_crc_verified']);self.assertEqual(result['missing_abstract_rows'],1)
+        duplicate=result['duplicate_ids'][0];self.assertEqual(duplicate['occurrences'],3)
+        self.assertTrue(duplicate['has_payload_variants']);self.assertEqual([v['occurrences'] for v in duplicate['variants']],[1,2])
+        with closing(sqlite3.connect(result['id_index'])) as c:self.assertEqual(c.execute('SELECT count(*) FROM raw_ids').fetchone()[0],2)
+        with self.assertRaisesRegex(ValueError,'output exists'):audit(self.archive,self.out)
+
+    def test_invalid_or_unverified_input_never_publishes_audit_outputs(self):
+        self.write('{"id":"2601.00001","title":"Valid"}\nnot JSON')
+        with self.assertRaises(json.JSONDecodeError):audit(self.archive,self.out)
+        self.assertFalse(self.out.exists());self.assertFalse(self.out.with_suffix('.ids.sqlite').exists())
+        self.write('{"id":"2601.00001","title":"Valid"}')
+        with self.assertRaisesRegex(ValueError,'checksum mismatch'):audit(self.archive,self.out,'0'*64)
+        self.assertFalse(self.out.exists())
+
+
+if __name__=='__main__':unittest.main()

tests/test_arxiv_tag_audit.py

@@ -0,0 +1,138 @@
+import fcntl
+import json
+from pathlib import Path
+import tempfile
+import unittest
+import sqlite3
+from unittest.mock import patch
+
+from paper_atlas.audits.arxiv_tags import audit, main
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.arxiv import refresh_categories
+
+
+class ArxivTagAuditTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.db = self.root / 'papers.sqlite'
+        self.out = self.root / 'audit.json'
+        self.store = Store(self.db)
+        self.addCleanup(self.store.conn.close)
+        self.records = []
+        for external in ('one', 'two'):
+            paper, record = self.store.record('arxiv', external, 'https://arxiv.org/abs/' + external,
+                'Same research work', 'Abstract', [], {'doi': '10.1234/same'},
+                {'categories': 'cs.AI cs.LG cs.AI'})
+            refresh_categories(self.store, paper, record, 'cs.AI cs.LG cs.AI')
+            self.records.append(record)
+        self.paper = paper
+        tag = self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid
+        self.store.conn.execute("INSERT INTO paper_tags VALUES(?,?,NULL,'manual:1')", (paper, tag))
+        self.store.conn.commit()
+
+    def test_all_merged_records_keep_their_own_categories_and_custom_tags_are_untouched(self):
+        before = self.store.conn.execute('SELECT * FROM paper_tags ORDER BY evidence_key,tag_id').fetchall()
+        result = audit(self.db, self.out, batch_size=1)
+        self.assertEqual(result['status'], 'passed')
+        self.assertEqual(result['records_checked'], 2)
+        self.assertEqual(result['expected_evidence'], 4)
+        self.assertEqual(result['actual_evidence'], 4)
+        self.assertEqual(result['evidence_without_arxiv_record'], 0)
+        self.assertEqual(self.store.conn.execute('SELECT * FROM paper_tags ORDER BY evidence_key,tag_id').fetchall(), before)
+
+    def test_zero_legacy_keys_does_not_hide_missing_categories_and_cli_fails(self):
+        self.store.conn.execute("DELETE FROM paper_tags WHERE source_record_id IS NOT NULL")
+        self.store.conn.commit()
+        with self.assertRaises(SystemExit):
+            main(['--db', str(self.db), '--out', str(self.out)])
+        result = json.loads(self.out.read_text())
+        self.assertEqual(result['status'], 'failed')
+        self.assertEqual(result['legacy_evidence'], 0)
+        self.assertEqual(result['missing_evidence'], 4)
+        self.assertEqual(result['records_with_mismatches'], 2)
+
+    def test_wrong_canonical_paper_or_stale_key_is_detected(self):
+        other, _ = self.store.record('example', 'other', 'https://example.org', 'Different work', 'AB', [])
+        self.store.conn.execute("UPDATE paper_tags SET paper_id=?,evidence_key='arxiv:categories' WHERE source_record_id=?", (other, self.records[0]))
+        self.store.conn.commit()
+        result = audit(self.db, self.out)
+        self.assertEqual(result['status'], 'failed')
+        self.assertEqual(result['records_with_mismatches'], 1)
+        self.assertEqual(result['legacy_evidence'], 2)
+        self.assertEqual(result['missing_evidence'], 2)
+        self.assertEqual(result['unexpected_evidence'], 2)
+
+    def test_wrong_source_and_unowned_category_rows_cannot_escape_the_full_audit(self):
+        paper, record = self.store.record('example', 'other', 'https://example.org', 'Other work', 'AB', [])
+        tag = self.store.conn.execute("SELECT id FROM tags WHERE namespace='arxiv:category' AND value='cs.AI'").fetchone()[0]
+        self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)', (paper, tag, record, 'wrong-source'))
+        self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)', (paper, tag, 'unowned'))
+        self.store.conn.commit()
+        result = audit(self.db, self.out)
+        self.assertEqual(result['status'], 'failed')
+        self.assertEqual(result['records_with_mismatches'], 0)
+        self.assertEqual(result['evidence_without_arxiv_record'], 2)
+        self.assertEqual(len(result['unscoped_examples']), 2)
+
+    def test_active_writer_and_incomplete_repair_are_rejected_before_publishing(self):
+        with Path(str(self.db) + '.maintenance.lock').open('a') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            with self.assertRaises(BlockingIOError):
+                audit(self.db, self.out)
+        self.assertFalse(self.out.exists())
+        repair = self.root / 'repair.json'
+        repair.write_text(json.dumps({'status': 'running'}))
+        with self.assertRaisesRegex(ValueError, 'not completed'):
+            audit(self.db, self.out, repair)
+        self.assertFalse(self.out.exists())
+
+    def test_repair_report_scope_and_completed_audit_are_not_silently_reused(self):
+        repair = self.root / 'repair.json'
+        repair.write_text(json.dumps({'status': 'complete', 'records_refreshed': 2, 'db': str(self.db),
+            'source_id': 1, 'source_record_count': 2, 'maximum_source_record_id': 999}))
+        with self.assertRaisesRegex(ValueError, 'different source inventory'):
+            audit(self.db, self.out, repair)
+        result = audit(self.db, self.out)
+        before = self.out.read_bytes()
+        with self.assertRaisesRegex(ValueError, 'report exists'):
+            audit(self.db, self.out)
+        self.assertEqual(self.out.read_bytes(), before)
+        self.assertEqual(result['status'], 'passed')
+
+    def test_invalid_source_metadata_is_an_error_not_a_partial_pass(self):
+        self.store.conn.execute('UPDATE source_records SET metadata_json=? WHERE id=?',
+                                (json.dumps({'categories': ['invalid']}), self.records[0]))
+        self.store.conn.commit()
+        with self.assertRaisesRegex(ValueError, 'categories must be text'):
+            audit(self.db, self.out)
+        self.assertEqual(json.loads(self.out.read_text())['status'], 'error')
+
+    def test_interleaved_sources_use_bounded_primary_key_batches_without_sorting(self):
+        self.store.record('example', 'interleaved', 'https://example.org', 'Other source', 'AB', [])
+        paper, record = self.store.record('arxiv', 'three', 'https://arxiv.org/abs/three',
+                                          'A third record', 'AB', [], metadata={'categories': 'cs.AI'})
+        refresh_categories(self.store, paper, record, 'cs.AI')
+        self.store.conn.commit()
+        queries = []
+        connect = sqlite3.connect
+        def traced(*args, **kwargs):
+            connection = connect(*args, **kwargs)
+            connection.set_trace_callback(queries.append)
+            return connection
+        with patch('paper_atlas.audits.arxiv_tags.sqlite3.connect', side_effect=traced):
+            result = audit(self.db, self.out, batch_size=1)
+        self.assertEqual(result['status'], 'passed')
+        self.assertEqual(result['records_checked'], 3)
+        self.assertEqual(result['last_record_id'], record)
+        batches = [sql for sql in queries if sql.startswith('SELECT id,external_id,paper_id,metadata_json')]
+        self.assertEqual(len(batches), 4)  # Includes the bounded final empty read.
+        for sql in batches:
+            plan = ' '.join(row[3] for row in self.store.conn.execute('EXPLAIN QUERY PLAN ' + sql))
+            self.assertIn('INTEGER PRIMARY KEY', plan)
+            self.assertNotIn('TEMP B-TREE', plan)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_arxiv_tags.py

@@ -0,0 +1,49 @@
+from contextlib import closing
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.arxiv_tags import repair
+
+
+class ArxivTagRepairTests(unittest.TestCase):
+    def setUp(self):
+        self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup);self.root=Path(self.temp.name)
+        self.db=self.root/'papers.sqlite';self.out=self.root/'repair.json'
+        disk=patch('paper_atlas.ingestion.arxiv_tags.shutil.disk_usage',return_value=SimpleNamespace(free=16*1024**3));disk.start();self.addCleanup(disk.stop)
+        store=Store(self.db)
+        for key in ('one','two'):
+            paper,record=store.record('arxiv',key,'https://arxiv.org/abs/'+key,'Same title','AB',[],{'doi':'10.1234/same'},{'categories':'cs.AI cs.LG'})
+            for category in ('cs.AI','cs.LG'):
+                store.conn.execute("INSERT OR IGNORE INTO tags(namespace,value) VALUES('arxiv:category',?)",(category,))
+                tag=store.conn.execute("SELECT id FROM tags WHERE namespace='arxiv:category' AND value=?",(category,)).fetchone()[0]
+                store.conn.execute("INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,'arxiv:categories')",(paper,tag,record))
+        tag=store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid
+        store.conn.execute("INSERT INTO paper_tags VALUES(?,?,NULL,'manual:1')",(paper,tag));store.conn.commit();store.conn.close()
+
+    def test_restore_collided_evidence_and_preserve_custom_tags_idempotently(self):
+        result=repair(self.db,self.out,batch_size=1)
+        self.assertEqual(result['records_refreshed'],2);self.assertEqual(result['status'],'complete')
+        with closing(sqlite3.connect(self.db)) as c:
+            self.assertEqual(c.execute('SELECT count(*) FROM paper_tags').fetchone()[0],5)
+            self.assertEqual(c.execute('SELECT count(DISTINCT source_record_id) FROM paper_tags').fetchone()[0],2)
+            self.assertEqual(c.execute("SELECT count(*) FROM paper_tags WHERE evidence_key='manual:1'").fetchone()[0],1)
+        repair(self.db,self.root/'again.json')
+        with closing(sqlite3.connect(self.db)) as c:self.assertEqual(c.execute('SELECT count(*) FROM paper_tags').fetchone()[0],5)
+
+    def test_failed_batch_resumes_only_after_last_committed_record(self):
+        with closing(sqlite3.connect(self.db)) as c,c:c.execute("UPDATE source_records SET metadata_json=? WHERE external_id='two'",(json.dumps({'categories':['invalid']}),))
+        with self.assertRaisesRegex(ValueError,'categories must be text'):repair(self.db,self.out,batch_size=1)
+        state=json.loads(self.out.read_text());self.assertEqual(state['records_refreshed'],1);self.assertEqual(state['status'],'failed')
+        with closing(sqlite3.connect(self.db)) as c,c:c.execute("UPDATE source_records SET metadata_json=? WHERE external_id='two'",(json.dumps({'categories':'cs.AI cs.LG'}),))
+        result=repair(self.db,self.out,resume=True,batch_size=1)
+        self.assertEqual(result['records_refreshed'],2);self.assertEqual(result['status'],'complete')
+        with closing(sqlite3.connect(self.db)) as c:self.assertEqual(c.execute('SELECT count(*) FROM paper_tags').fetchone()[0],5)
+
+
+if __name__=='__main__':unittest.main()

tests/test_cli.py

@@ -0,0 +1,35 @@
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+from paper_atlas.cli import COMMANDS
+
+
+class CliTests(unittest.TestCase):
+    def test_every_registered_command_has_help_without_starting_work(self):
+        for route in COMMANDS:
+            with self.subTest(route=route):
+                result=subprocess.run([sys.executable,'-B','-m','paper_atlas',*route,'--help'],capture_output=True,text=True)
+                self.assertEqual(result.returncode,0,result.stderr)
+                self.assertIn('paper-atlas '+' '.join(route),result.stdout)
+
+    def test_unified_import_and_query_use_shipped_schema_and_registry(self):
+        from paper_atlas.config import DEFAULT_REGISTRY
+        venues = json.loads(DEFAULT_REGISTRY.read_text())['venues']
+        names = [venue['acronym'] for venue in venues]
+        self.assertTrue({'COLM', 'MM', 'MLSys'} <= set(names))
+        self.assertEqual(len(names), len(set(names)))
+        with tempfile.TemporaryDirectory() as tmp:
+            root=Path(tmp);db=root/'db.sqlite';source=root/'records.jsonl'
+            source.write_text(json.dumps({'source':'example','external_id':'one','url':'https://example.org','title':'CLI paper','abstract':'Abstract','authors':['Jane Doe'],'venue':'Example','year':2026,'track':'main','status':'accepted'})+'\n')
+            source.with_suffix('.manifest.json').write_text(json.dumps({'source':'example','list_url':'https://example.org','count':1}))
+            for args in [('import','records','--db',str(db),str(source)),('query','--db',str(db),'--stats')]:
+                result=subprocess.run([sys.executable,'-B','-m','paper_atlas',*args],capture_output=True,text=True)
+                self.assertEqual(result.returncode,0,result.stderr)
+            self.assertEqual(json.loads(result.stdout)['papers'],1)
+
+
+if __name__=='__main__':unittest.main()

tests/test_coverage_report.py

@@ -3,8 +3,8 @@ from pathlib import Path
 import tempfile
 import unittest
 
-from coverage_report import report
-from index_core import Store
+from paper_atlas.audits.coverage import report
+from paper_atlas.db.store import Store
 
 
 class CoverageTests(unittest.TestCase):

tests/test_dblp_catalogue.py

@@ -6,7 +6,7 @@ import sqlite3
 import tempfile
 import unittest
 
-from catalogue_dblp import catalogue,records
+from paper_atlas.collectors.dblp import catalogue,records
 
 
 class DblpCatalogueTests(unittest.TestCase):

tests/test_dblp_records.py

@@ -0,0 +1,145 @@
+import hashlib
+import json
+from contextlib import closing
+from pathlib import Path
+import sqlite3
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors.dblp import XML_SHA
+from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row,material_reason
+from paper_atlas.collectors.dblp_scope import scope
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.records import ingest_file
+
+
+class DblpRecordTests(unittest.TestCase):
+    def setUp(self):
+        # Tiny fixtures may live on a small tmpfs; production reserve stays 8 GiB.
+        budget=patch('paper_atlas.collectors.dblp_records.shutil.disk_usage',return_value=SimpleNamespace(free=16*1024**3))
+        budget.start();self.addCleanup(budget.stop)
+
+    def row(self,family='aaai',book='AAAI',year=2024,key='one'):
+        return {'record_type':'inproceedings','key':f'conf/{family}/{key}','publtype':None,
+                'fields':{'title':['A research publication'],'author':['Jane Doe 0001','John Smith'],
+                          'booktitle':[book],'year':[str(year)],'crossref':[f'conf/{family}/{year}'],'ee':['https://doi.org/10.1234/'+key]}}
+
+    def volume(self,family,year,title=None):
+        return {'record_type':'proceedings','key':f'conf/{family}/{year}','fields':{'title':[title or f'{family.upper()} {year}'],'year':[str(year)]}}
+
+    def test_real_directory_aliases_do_not_confuse_other_conferences(self):
+        row=self.row('clear2','CLeaR');volume=self.volume('clear2',2024,'Causal Learning and Reasoning, 2024')
+        self.assertEqual(scope(row,volume)[0]['venue'],'CLeaR')
+        self.assertIsNone(scope(self.row('clear','CLEAR'),self.volume('clear',2024))[0])
+        self.assertIsNone(scope(self.row('aamas','Adaptive Agents and Multi-Agent Systems'),self.volume('aamas',2024))[0])
+        self.assertEqual(scope(self.row('atal','AAMAS'),self.volume('atal',2024))[0]['venue'],'AAMAS')
+        self.assertEqual(scope(self.row('aips','ICAPS'),self.volume('aips',2024))[0]['venue'],'ICAPS')
+
+    def test_conference_year_is_distinct_from_publication_year(self):
+        row=self.row('nips','NIPS',1988);row['fields']['crossref']=['conf/nips/1987']
+        volume=self.volume('nips',1987,'NIPS Conference, Denver, 1987; published 1988')
+        result,_=scope(row,volume);self.assertEqual(result['year'],1987);self.assertEqual(result['bibliographic_year'],1988)
+        volume['fields']['title']=['NIPS collected papers']
+        self.assertEqual(scope(row,volume)[1],'unverified_year_difference')
+
+    def test_joint_events_and_tracks_are_explicit(self):
+        result,_=scope(self.row('ijcai','IJCAI/ECAI',2022),self.volume('ijcai',2022))
+        self.assertEqual(result['venue'],'ECAI+IJCAI');self.assertEqual(result['venue_aliases'],['ECAI','IJCAI'])
+        result,_=scope(self.row('coling','LREC/COLING'),self.volume('coling',2024))
+        self.assertEqual(result['venue'],'COLING+LREC')
+        for book,track in [('ACL (Findings)','findings'),('ACL (Student Research Workshop)','workshop'),('ACL (System Demonstrations)','demo')]:
+            self.assertEqual(scope(self.row('acl',book),self.volume('acl',2024))[0]['track'],track)
+        for suffix,track in [('Special interest tracks and posters','special_interest'),('Alternate Track Papers & Posters','alternate')]:
+            self.assertEqual(scope(self.row('www','WWW ('+suffix+')'),self.volume('www',2024))[0]['track'],track)
+        volume=self.volume('pkdd',2026,'ECML PKDD 2026, Applied Data Science Track, Demo Track and Industrial Track')
+        self.assertEqual(scope(self.row('pkdd','ECML PKDD (10)',2026),volume)[0]['track'],'proceedings_unspecified')
+
+    def test_short_year_keys_require_a_unique_explicit_year_not_a_guessed_century(self):
+        row=self.row('sigir','SIGIR',1980);volume=self.volume('sigir',1980,'Information Retrieval Research, Cambridge, June 1980')
+        volume['key']='conf/sigir/80';volume['fields']['year']=['1981'];row['fields']['crossref']=[volume['key']]
+        edition,_=scope(row,volume);self.assertEqual(edition['year'],1980)
+        self.assertIn('two-digit',edition['edition_year_basis'])
+        volume['fields']['title']=['SIGIR 1980, published 1981']
+        self.assertEqual(scope(row,volume)[1],'unverified_edition_year')
+        volume['fields']['title']=['SIGIR 1989']
+        self.assertEqual(scope(row,volume)[1],'unverified_edition_year')
+
+    def test_crossref_mismatch_and_future_editions_are_held_for_review(self):
+        self.assertEqual(scope(self.row(),self.volume('kdd',2024))[1],'crossref_venue_conflict')
+        self.assertEqual(scope(self.row(year=2027),self.volume('aaai',2027))[1],'future_edition')
+
+    def test_only_own_identifier_links_are_used_and_ambiguity_is_preserved(self):
+        own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/published','https://doi.org/10.48550/arXiv.2401.00001','https://openreview.net/forum?id=realForum'],
+                                    'cite':['https://doi.org/10.9999/reference']})
+        self.assertEqual(own,{'doi':'10.1234/published','arxiv':'2401.00001','openreview':'realForum'});self.assertFalse(ambiguous)
+        own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/a','https://doi.org/10.1234/b','https://openreview.net/forum?id=2026-Oral--99-placeholder']})
+        self.assertEqual(own,{});self.assertEqual(len(ambiguous['doi']),2)
+
+    def test_frontmatter_and_anonymous_event_headers_are_not_paper_acceptances(self):
+        for title in ('Title, Preface.','Title, Conference Committee, Table of Contents, Author Index.','Front Matter.','Backmatter.','Foreword.'):
+            self.assertEqual(material_reason(title,['An Editor'],'conf/sigir/C78',1978,'main'),'frontmatter')
+        for title in ('Sponsors.','IJCAI-09 Awards and Distinguished Papers.','2013 Conference Organization.','Area Chairs and Program Committee.','Welcome to ICDM 2005.','Workshop Summaries.'):
+            self.assertIsNotNone(material_reason(title,[],'conf/ijcai/X09',2009,'main'))
+        self.assertEqual(material_reason("Wiki Workshop Chairs' Welcome.",['An Organizer'],'conf/www/X19t',2019,'companion'),'frontmatter')
+        self.assertEqual(material_reason('Attention - from Neuroscience to the Web and Wellbeing.',[],'conf/www/X19b',2019,'companion'),'unverified_proceedings_material')
+        row=self.row();row['fields']['title']=['Sponsors.'];row['fields']['author']=[]
+        self.assertEqual(normalize_row(row,{'conf/aaai/2024':self.volume('aaai',2024)},{'AAAI'})[1],'proceedings_heading')
+
+    def test_missing_authors_alone_do_not_remove_real_research_records(self):
+        self.assertIsNone(material_reason('The Onomastica interlanguage pronunciation lexicon.',[],'conf/interspeech/X95',1995,'main'))
+        self.assertIsNone(material_reason('Learning Models of Relational Stochastic Processes.',[],'conf/ecml/SanghaiDW05',2005,'main'))
+        self.assertIsNone(material_reason('Sponsors.', ['A Researcher'],'conf/aaai/research',2024,'main'))
+
+    def fixture(self,root,rows):
+        catalogue=root/'dblp-catalogue.sqlite';volume=self.volume('aaai',2024)
+        with closing(sqlite3.connect(catalogue)) as c, c:
+            c.execute('CREATE TABLE proceedings(key TEXT PRIMARY KEY,metadata_json TEXT)')
+            c.execute('INSERT INTO proceedings VALUES(?,?)',(volume['key'],json.dumps(volume)))
+        candidates=root/'candidates.jsonl';candidates.write_text(''.join(json.dumps(row)+'\n' for row in rows))
+        manifest={'snapshot_sha256':XML_SHA,'candidate_sha256':hashlib.sha256(candidates.read_bytes()).hexdigest(),
+                  'candidate_records':len(rows),'record_types':{'proceedings':1,'inproceedings':len(rows)}}
+        catalogue.with_name('dblp-ai-candidates.manifest.json').write_text(json.dumps(manifest))
+        return catalogue,candidates
+
+    def test_normalized_files_import_with_unknown_abstracts_and_no_invented_retracted_acceptance(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root=Path(tmp);withdrawn=self.row(key='two');withdrawn['publtype']='withdrawn'
+            catalogue,candidates=self.fixture(root,[self.row(),withdrawn,self.row(book='AAAI Workshops',key='workshop')])
+            result=collect(catalogue,root/'out',['AAAI'],candidates=candidates)
+            self.assertEqual(result['normalized_records'],2);self.assertEqual(result['excluded_by_reason']['separate_workshop'],1)
+            manifest=result['files'][0];self.assertEqual(manifest['missing_abstract_count'],2)
+            rows=[json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
+            self.assertEqual(rows[0]['authors'],['Jane Doe','John Smith']);self.assertIsNone(rows[0]['abstract'])
+            self.assertEqual(rows[1]['status'],'unknown');self.assertEqual(rows[1]['decision_events'],[])
+            db=root/'papers.sqlite';ingest_file(db,manifest['file'])
+            with closing(sqlite3.connect(db)) as c:self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],1)
+
+    def test_checksum_failure_and_duplicate_keys_do_not_publish_files(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root=Path(tmp);catalogue,candidates=self.fixture(root,[self.row(),self.row()])
+            with self.assertRaisesRegex(ValueError,'Repeated DBLP'):collect(catalogue,root/'out',['AAAI'],candidates=candidates)
+            self.assertFalse(list((root/'out').glob('*.jsonl')))
+            candidates.write_text('tampered')
+            with self.assertRaisesRegex(ValueError,'checksum mismatch'):collect(catalogue,root/'out',['AAAI'],candidates=candidates)
+
+    def test_case_variants_use_one_conference_edition_and_submission(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store=Store(Path(tmp)/'papers.sqlite');self.addCleanup(store.conn.close)
+            paper,one=store.record('pmlr','one','https://example.org/one','A paper','Abstract',[],{'doi':'10.1234/one'})
+            store.decision(paper,one,'CORL',2024,'main','one','https://example.org/one','accepted','Accept',identities=[('publication_doi','10.1234/one')])
+            same,two=store.record('dblp','two','https://example.org/two','A paper',None,[],{'doi':'10.1234/one'})
+            store.decision(same,two,'CoRL',2024,'main','two','https://example.org/two','accepted','Published',identities=[('publication_doi','10.1234/one')])
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM venues').fetchone()[0],1)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
+
+    def test_production_disk_reserve_stops_before_publishing(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root=Path(tmp);catalogue,candidates=self.fixture(root,[self.row()])
+            with patch('paper_atlas.collectors.dblp_records.shutil.disk_usage',return_value=SimpleNamespace(free=1024)):
+                with self.assertRaisesRegex(RuntimeError,'8 GiB'):collect(catalogue,root/'out',['AAAI'],candidates=candidates)
+            self.assertFalse(list((root/'out').glob('*.jsonl')))
+
+
+if __name__=='__main__':unittest.main()

tests/test_download_snapshot.py

@@ -6,7 +6,7 @@ import tempfile
 import unittest
 from unittest.mock import patch
 
-from download_snapshot import download
+from paper_atlas.io.snapshots import download
 
 
 class Response(io.BytesIO):
@@ -19,16 +19,16 @@ class DownloadTests(unittest.TestCase):
         body=b'public metadata';sha=hashlib.sha256(body).hexdigest()
         with tempfile.TemporaryDirectory() as tmp:
             path=Path(tmp)/'snapshot.gz'
-            with patch('download_snapshot.urllib.request.urlopen',return_value=Response(body)):
+            with patch('paper_atlas.io.snapshots.urllib.request.urlopen',return_value=Response(body)):
                 result=download('https://example.org/snapshot.gz',path,sha256=sha,expected_bytes=len(body),min_free=0)
             self.assertEqual(result['status'],'verified')
-            with patch('download_snapshot.urllib.request.urlopen',side_effect=AssertionError('No network expected')):
+            with patch('paper_atlas.io.snapshots.urllib.request.urlopen',side_effect=AssertionError('No network expected')):
                 reused=download('https://example.org/snapshot.gz',path,sha256=sha,min_free=0)
             self.assertEqual(reused['status'],'already_verified')
             self.assertEqual(json.loads(Path(str(path)+'.manifest.json').read_text())['sha256'],sha)
 
     def test_bad_checksum_never_publishes_a_snapshot(self):
-        with tempfile.TemporaryDirectory() as tmp,patch('download_snapshot.urllib.request.urlopen',return_value=Response(b'wrong bytes')):
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.io.snapshots.urllib.request.urlopen',return_value=Response(b'wrong bytes')):
             path=Path(tmp)/'snapshot.gz'
             with self.assertRaisesRegex(ValueError,'SHA256 mismatch'):
                 download('https://example.org/snapshot.gz',path,sha256='0'*64,min_free=0)
@@ -41,16 +41,16 @@ class DownloadTests(unittest.TestCase):
             Path(str(part)+'.json').write_text(json.dumps({'url':url,'expected_sha256':sha,'expected_md5':None,'expected_bytes':6}))
             with self.assertRaisesRegex(ValueError,'same snapshot'):
                 download(url,path,sha256=sha,expected_bytes=6,min_free=0)
-            with patch('download_snapshot.urllib.request.urlopen',return_value=Response(body)):
+            with patch('paper_atlas.io.snapshots.urllib.request.urlopen',return_value=Response(body)):
                 with self.assertRaisesRegex(ValueError,'resume range'):
                     download(url,path,sha256=sha,expected_bytes=6,resume=True,min_free=0)
             self.assertEqual(part.read_bytes(),b'abc')
-            with patch('download_snapshot.urllib.request.urlopen',return_value=Response(b'def',206,{'Content-Range':'bytes 3-5/6'})):
+            with patch('paper_atlas.io.snapshots.urllib.request.urlopen',return_value=Response(b'def',206,{'Content-Range':'bytes 3-5/6'})):
                 download(url,path,sha256=sha,expected_bytes=6,resume=True,min_free=0)
             self.assertEqual(path.read_bytes(),body)
 
     def test_wrong_http_length_is_rejected_before_writing(self):
-        with tempfile.TemporaryDirectory() as tmp,patch('download_snapshot.urllib.request.urlopen',return_value=Response(b'abc')):
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.io.snapshots.urllib.request.urlopen',return_value=Response(b'abc')):
             path=Path(tmp)/'snapshot.gz'
             with self.assertRaisesRegex(ValueError,'HTTP size'):
                 download('https://example.org/snapshot.gz',path,sha256='0'*64,expected_bytes=4,min_free=0)
@@ -61,7 +61,7 @@ class DownloadTests(unittest.TestCase):
         with tempfile.TemporaryDirectory() as tmp:
             path=Path(tmp)/'snapshot.gz';part=Path(str(path)+'.part');part.write_bytes(b'')
             Path(str(part)+'.json').write_text(json.dumps({'url':url,'expected_sha256':sha,'expected_md5':None,'expected_bytes':3}))
-            with patch('download_snapshot.urllib.request.urlopen',return_value=Response(body)):
+            with patch('paper_atlas.io.snapshots.urllib.request.urlopen',return_value=Response(body)):
                 download(url,path,sha256=sha,expected_bytes=3,resume=True,min_free=0)
             metadata=Path(str(path)+'.manifest.json');metadata.unlink()
             download(url,path,sha256=sha,expected_bytes=3,min_free=0)

tests/test_fetch.py

@@ -6,7 +6,7 @@ import unittest
 from unittest.mock import patch
 from urllib.error import HTTPError,URLError
 
-from collect_pmlr import fetch
+from paper_atlas.io.http import fetch
 
 
 class Response(io.BytesIO):
@@ -16,30 +16,30 @@ class Response(io.BytesIO):
 
 class FetchTests(unittest.TestCase):
     def test_transient_connection_failure_is_retried(self):
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.time.sleep'),patch('collect_pmlr.urllib.request.urlopen',side_effect=[URLError(ConnectionResetError('reset')),Response(b'complete')]) as network:
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.io.http.time.sleep'),patch('paper_atlas.io.http.urllib.request.urlopen',side_effect=[URLError(ConnectionResetError('reset')),Response(b'complete')]) as network:
             self.assertEqual(fetch('https://example.org/metadata',tmp),b'complete')
             self.assertEqual(network.call_count,2)
 
     def test_truncated_response_is_not_cached_as_complete(self):
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.time.sleep'),patch('collect_pmlr.urllib.request.urlopen',side_effect=[Response(b'bad',expected=8),Response(b'complete')]):
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.io.http.time.sleep'),patch('paper_atlas.io.http.urllib.request.urlopen',side_effect=[Response(b'bad',expected=8),Response(b'complete')]):
             self.assertEqual(fetch('https://example.org/metadata',tmp),b'complete')
             path=Path(tmp)/hashlib.sha256(b'https://example.org/metadata').hexdigest()
             self.assertEqual(path.read_bytes(),b'complete')
 
     def test_forbidden_response_is_not_retried_or_bypassed(self):
         body=io.BytesIO(b'Forbidden');error=HTTPError('https://example.org/metadata',403,'Forbidden',{},body)
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.time.sleep'),patch('collect_pmlr.urllib.request.urlopen',side_effect=error) as network:
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.io.http.time.sleep'),patch('paper_atlas.io.http.urllib.request.urlopen',side_effect=error) as network:
             with self.assertRaises(HTTPError):fetch('https://example.org/metadata',tmp)
             self.assertEqual(network.call_count,1)
             self.assertTrue(body.closed)
 
     def test_transport_retries_are_bounded(self):
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.time.sleep'),patch('collect_pmlr.urllib.request.urlopen',side_effect=ConnectionResetError('reset')) as network:
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.io.http.time.sleep'),patch('paper_atlas.io.http.urllib.request.urlopen',side_effect=ConnectionResetError('reset')) as network:
             with self.assertRaises(ConnectionResetError):fetch('https://example.org/metadata',tmp)
             self.assertEqual(network.call_count,3)
 
     def test_cached_response_still_obeys_the_size_limit(self):
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.urllib.request.urlopen') as network:
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.io.http.urllib.request.urlopen') as network:
             path=Path(tmp)/hashlib.sha256(b'https://example.org/metadata').hexdigest();path.write_bytes(b'too large')
             with self.assertRaisesRegex(RuntimeError,'exceeds byte limit'):
                 fetch('https://example.org/metadata',tmp,limit=2)

tests/test_iclr.py

@@ -5,9 +5,9 @@ import unittest
 from unittest.mock import patch
 from urllib.error import HTTPError
 
-from collect_iclr import asset_urls, collect_year, discover, parse_program
-from import_records import ingest_file
-from index_core import Store
+from paper_atlas.collectors.iclr import asset_urls, collect_year, discover, parse_program
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.db.store import Store
 
 
 class ICLRTests(unittest.TestCase):
@@ -64,6 +64,7 @@ class ICLRTests(unittest.TestCase):
         rows,_=self.parse([dict(row,paper_pdf_url=None)],year=2020)
         self.assertEqual(rows[0]['metadata']['identity_state'],'program_only')
         self.assertEqual(rows[0]['identifiers'],{})
+        self.assertEqual(rows[0]['url'],'https://iclr.cc/virtual/2020/papers.html')
 
     def test_only_referenced_official_assets_and_real_navigation_years(self):
         html='<a href="/archive/2013/">2013</a><a href="/Conferences/2026">2026</a><a href="https://other.example/2025">2025</a>'
@@ -75,7 +76,7 @@ class ICLRTests(unittest.TestCase):
         html=b'start("/static/virtual/data/iclr-2021-orals-posters.json",true,"/static/virtual/data/iclr-2021-abstracts.json")'
         event=self.event(sourceurl=None,decision='Poster',abstract='Embedded historical abstract')
         responses=[html,json.dumps({'count':1,'results':[event]}).encode(),HTTPError('https://iclr.cc/abstracts',404,'Not Found',None,None)]
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_iclr.fetch',side_effect=responses):
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.iclr.fetch',side_effect=responses):
             manifest=collect_year(2021,tmp,tmp)
             self.assertEqual(manifest['status'],'collected')
             self.assertEqual(manifest['missing_abstract_count'],0)

tests/test_iclr_legacy.py

@@ -0,0 +1,143 @@
+import copy
+import hashlib
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.audits import iclr_legacy
+from paper_atlas.ingestion.backfill import backfill, queue_files
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.db.store import Store
+
+
+class ICLRLegacyTests(unittest.TestCase):
+    def program(self):
+        return {'source': 'iclr_program', 'external_id': '2020/program-42', 'venue': 'ICLR',
+                'year': 2020, 'track': 'main', 'status': 'accepted',
+                'url': 'https://iclr.cc/virtual/2020/papers.html', 'title': 'Historical paper title',
+                'abstract': 'Historical abstract', 'authors': ['Jane Doe', 'John Smith'],
+                'identifiers': {}, 'raw_decision': 'listed in official program',
+                'evidence_url': 'https://iclr.cc/virtual/2020/papers.html',
+                'metadata': {'program_identity_type': 'submission', 'program_source_id': 42}}
+
+    def snapshot(self):
+        return {'source': 'openreview_snapshot', 'external_id': '2020/realForum', 'venue': 'ICLR',
+                'year': 2020, 'track': 'main', 'status': 'accepted',
+                'url': 'https://openreview.net/forum?id=realForum', 'title': 'Historical paper title',
+                'abstract': 'Historical abstract', 'authors': ['Jane Doe', 'John Smith'],
+                'identifiers': {'openreview': 'realForum'},
+                'submission_identifiers': [{'scheme': 'openreview', 'value': 'realForum'},
+                                           {'scheme': 'iclr_number', 'value': '42'}],
+                'decision_events': [{'status': 'accepted', 'raw_value': 'Accept',
+                    'source_decision_id': 'decision1', 'evidence_url': 'https://openreview.net/forum?id=realForum&noteId=decision1',
+                    'authority': {'public_readers_verified': True}}],
+                'metadata': {'source_submission_number': 42, 'public_readers_verified': True}}
+
+    def legacy(self):
+        return {'id': 'realForum', 'forum': 'realForum', 'content': {
+            'iclr_id': 'realForum', 'title': 'Historical paper title', 'authors': ['Jane Doe', 'John Smith']}}
+
+    def test_independent_legacy_identity_preserves_existing_source_id_and_text(self):
+        program, submission = self.program(), self.snapshot()
+        submission.update(title='A later title', abstract='A later abstract', authors=['Jane Doe'])
+        rows, checks = iclr_legacy.reconcile([program], {'realForum': submission}, [self.legacy()])
+        self.assertEqual(rows[0]['external_id'], '2020/program-42')
+        self.assertEqual(rows[0]['url'], 'https://iclr.cc/virtual_2020/poster_realForum.html')
+        for field in ('title', 'abstract', 'authors'):
+            self.assertEqual(rows[0][field], program[field])
+        self.assertEqual(rows[0]['identifiers']['openreview'], 'realForum')
+        self.assertTrue(rows[0]['metadata']['program_submission_number_verified'])
+        self.assertEqual(checks['independent_legacy'], 1)
+        self.assertEqual(program['identifiers'], {})
+
+    def test_program_number_and_shared_authors_alone_do_not_assign_a_forum(self):
+        program = self.program()
+        program.update(title='A different paper', abstract='An unrelated abstract')
+        with self.assertRaisesRegex(ValueError, 'number alone'):
+            iclr_legacy.reconcile([program], {'realForum': self.snapshot()}, [self.legacy()])
+
+    def test_independent_snapshot_title_and_abstract_handle_legacy_name_variants(self):
+        legacy = self.legacy()
+        legacy['content']['authors'] = ['J. Doe', 'J. Smith']
+        rows, checks = iclr_legacy.reconcile([self.program()], {'realForum': self.snapshot()}, [legacy])
+        self.assertEqual(checks['independent_snapshot'], 1)
+        self.assertEqual(rows[0]['identifiers']['openreview'], 'realForum')
+
+    def test_reviewed_title_variant_is_exact_and_requires_public_author_corroboration(self):
+        variant = iclr_legacy.REVIEWED_TITLE_VARIANT
+        program, submission, legacy = self.program(), self.snapshot(), self.legacy()
+        program['title'] = variant['program']
+        submission.update(title=variant['snapshot'], abstract='A different revision')
+        submission['identifiers']['openreview'] = variant['forum']
+        legacy.update(id=variant['forum'], forum=variant['forum'])
+        legacy['content'].update(iclr_id=variant['forum'], title=variant['legacy'])
+        rows, checks = iclr_legacy.reconcile([program], {variant['forum']: submission}, [legacy])
+        self.assertEqual(checks['reviewed_title_variant'], 1)
+        self.assertTrue(rows[0]['metadata']['program_identity_verification']['reviewed_title_variant'])
+        program['authors'] = ['Jane Doe']
+        with self.assertRaisesRegex(ValueError, 'number alone'):
+            iclr_legacy.reconcile([program], {variant['forum']: submission}, [legacy])
+
+    def test_conflicting_forum_ambiguous_number_and_repeated_legacy_identity_fail(self):
+        p = self.program()
+        p['identifiers']['openreview'] = 'anotherForum'
+        with self.assertRaisesRegex(ValueError, 'conflicts'):
+            iclr_legacy.reconcile([p], {'realForum': self.snapshot()}, [self.legacy()])
+        duplicate = copy.deepcopy(self.snapshot())
+        duplicate['identifiers']['openreview'] = 'otherForum'
+        with self.assertRaisesRegex(ValueError, 'ambiguous'):
+            iclr_legacy.reconcile([self.program()], {'realForum': self.snapshot(), 'otherForum': duplicate}, [self.legacy()])
+        with self.assertRaisesRegex(ValueError, 'repeated'):
+            iclr_legacy.reconcile([self.program()], {'realForum': self.snapshot()}, [self.legacy(), self.legacy()])
+
+    def write_file(self, path, rows, source):
+        path.write_text(''.join(json.dumps(row) + '\n' for row in rows))
+        manifest = {'source': source, 'venue': 'ICLR', 'year': 2020, 'track': 'main',
+                    'parser_version': 2, 'count': len(rows), 'missing_abstract_count': 0,
+                    'list_url': 'https://iclr.cc/virtual/2020/papers.html', 'status': 'collected',
+                    'submission_status_counts': {status: sum(r['status'] == status for r in rows)
+                                                 for status in {r['status'] for r in rows}}}
+        path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
+
+    def test_audited_bridge_then_snapshot_import_keeps_one_accepted_attempt(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            program_path, snapshot_path, legacy_path = (root / name for name in ('program.jsonl', 'snapshot.jsonl', 'legacy.json'))
+            self.write_file(program_path, [self.program()], 'iclr_program')
+            self.write_file(snapshot_path, [self.snapshot()], 'openreview_snapshot')
+            raw = json.dumps([self.legacy()]).encode()
+            legacy_path.write_bytes(raw)
+            with patch.multiple(iclr_legacy, LEGACY_BYTES=len(raw), LEGACY_SHA256=hashlib.sha256(raw).hexdigest(), EXPECTED=1):
+                report = iclr_legacy.audit(program_path, snapshot_path, legacy_path, root / 'out')
+            db = root / 'papers.sqlite'
+            ingest_file(db, program_path)
+            queue = root / 'out/iclr-2020-backfill-queue.json'
+            result = backfill(db, queue_files(queue))
+            self.assertEqual(result['submissions_before'], result['submissions_after'])
+            ingest_file(db, report['backfill_queue']['files'][0])
+            ingest_file(db, snapshot_path)
+            store = Store(db)
+            try:
+                self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+                self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0], 1)
+                self.assertEqual(store.conn.execute('SELECT count(*) FROM submission_sources').fetchone()[0], 2)
+                self.assertEqual(store.conn.execute('SELECT url FROM source_records WHERE external_id=?', ('2020/program-42',)).fetchone()[0],
+                                 'https://iclr.cc/virtual_2020/poster_realForum.html')
+                self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(), [])
+            finally:
+                store.conn.close()
+
+    def test_unverified_legacy_bytes_publish_no_output(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            legacy = root / 'legacy.json'
+            legacy.write_text('[]')
+            with self.assertRaisesRegex(ValueError, 'checksum'):
+                iclr_legacy.audit(root / 'program', root / 'snapshot', legacy, root / 'out')
+            self.assertFalse((root / 'out').exists())
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_iclr_snapshot_audit.py

@@ -3,7 +3,7 @@ from pathlib import Path
 import tempfile
 import unittest
 
-from audit_iclr_snapshot import audit
+from paper_atlas.audits.iclr import audit
 
 
 class SnapshotAuditTests(unittest.TestCase):

tests/test_icml_public_archive.py

@@ -0,0 +1,86 @@
+import hashlib
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors import icml_public_archive as archive
+from paper_atlas.audits.public_openreview import audit
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.records import ingest_file
+
+
+class ICMLPublicArchiveTests(unittest.TestCase):
+    def note(self, **changes):
+        return {'id': 'PublicRoot1', 'forum': 'PublicRoot1', 'readers': ['everyone'],
+                'domain': archive.DOMAIN, 'invitations': [archive.DOMAIN + '/-/Submission'],
+                'license': 'CC BY 4.0', 'cdate': 1769251630136,
+                'content': {'title': {'value': 'A public rejected submission'},
+                            'abstract': {'value': 'Public abstract'},
+                            'authors': {'value': ['Anonymous', 'Jane Doe']},
+                            'authorids': {'value': ['private@example.org']},
+                            'venue': {'value': 'Submitted to ICML 2026'},
+                            'venueid': {'value': archive.VENUEID},
+                            'primary_area': {'value': 'learning'},
+                            'pdf': {'value': '/private.pdf'}},
+                'details': {'reviews': ['not retained']}, **changes}
+
+    def test_explicit_platform_classification_requires_public_root_and_correct_scope(self):
+        original = self.note()
+        normalized = archive.normalize(original)
+        self.assertEqual(normalized['status'], 'rejected')
+        self.assertEqual(normalized['raw_decision'], archive.VENUEID)
+        self.assertEqual(normalized['metadata']['license'], 'CC BY 4.0')
+        self.assertEqual(normalized['metadata']['evidence_kind'], 'public_submission_venueid')
+        self.assertFalse(normalized['metadata']['original_decision_signature_verified'])
+        self.assertNotIn('private@example.org', json.dumps(normalized))
+        self.assertNotIn('not retained', json.dumps(normalized))
+        for changes in ({'readers': ['authors']}, {'forum': 'AnotherRoot'}, {'ddate': 1000},
+                        {'domain': 'ICML.cc/2026/Position_Papers'}, {'invitations': []}):
+            with self.subTest(changes=changes), self.assertRaises(ValueError):
+                archive.normalize(self.note(**changes))
+        for venueid in (None, 'ICML.cc/2026/Conference/Submission', 'ICML.cc/2026/Conference/Accepted_Submission'):
+            candidate = self.note()
+            candidate['content']['venueid'] = {'value': venueid}
+            with self.assertRaisesRegex(ValueError, 'classification'):
+                archive.normalize(candidate)
+
+    def collect_fixture(self, out, notes, count=None, corrupt=False):
+        data = json.dumps({'count': len(notes) if count is None else count, 'notes': notes}).encode()
+        blob = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
+        with patch.multiple(archive, BYTES=len(data), BLOB_SHA1=blob, EXPECTED=len(notes)), \
+             patch.object(archive.shutil, 'disk_usage', return_value=type('Space', (), {'free': 20 * 1024**3})()):
+            return archive.collect(out, fetcher=lambda url, limit: data + b'x' if corrupt else data)
+
+    def test_verified_archive_roundtrips_through_preflight_and_idempotent_import(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            result = self.collect_fixture(root / 'source', [self.note()])
+            db = root / 'db.sqlite'
+            Store(db).conn.close()
+            report = audit(db, root / 'source/public-openreview-inventory.json', root / 'audit.json')
+            self.assertEqual(report['status'], 'ready_for_serial_import')
+            self.assertEqual(report['source'], archive.SOURCE)
+            self.assertEqual(result['files'][0]['status'], 'partial')
+            for _ in range(2):
+                ingest_file(db, report['files'][0])
+            store = Store(db)
+            try:
+                self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0], 1)
+                self.assertEqual(store.conn.execute('SELECT status FROM decisions').fetchall(), [('rejected',)])
+            finally:
+                store.conn.close()
+
+    def test_bad_hash_count_or_duplicate_identity_publishes_nothing(self):
+        for kwargs in ({'corrupt': True}, {'count': 99}, {}):
+            notes = [self.note(), self.note()] if not kwargs else [self.note()]
+            with self.subTest(kwargs=kwargs), tempfile.TemporaryDirectory() as tmp:
+                with self.assertRaises(ValueError):
+                    self.collect_fixture(tmp, notes, **kwargs)
+                self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
+                self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_import_arxiv.py

@@ -6,7 +6,9 @@ import unittest
 import zipfile
 from pathlib import Path
 
-from import_arxiv import ingest
+from paper_atlas.ingestion.arxiv import ingest
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.arxiv import refresh_categories
 
 
 class ImportTests(unittest.TestCase):
@@ -53,6 +55,28 @@ class ImportTests(unittest.TestCase):
         with closing(sqlite3.connect(self.db)) as c, c:
             self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON pt.tag_id=t.id WHERE t.value='cs.AI'").fetchone()[0],2)
 
+    def test_merged_records_retain_separate_category_evidence_across_refresh(self):
+        one=self.row('2601.00001','10.0000/example');two=self.row('2601.00002','10.0000/example')
+        self.write([one,two]);ingest(self.db,self.archive)
+        with closing(sqlite3.connect(self.db)) as c:
+            self.assertEqual(c.execute('SELECT count(*) FROM paper_tags').fetchone()[0],4)
+            self.assertEqual(c.execute('SELECT count(DISTINCT source_record_id) FROM paper_tags').fetchone()[0],2)
+        one['categories']='cs.AI';two['categories']='cs.LG';self.write([one,two]);ingest(self.db,self.archive)
+        with closing(sqlite3.connect(self.db)) as c:
+            actual=c.execute('SELECT r.external_id,t.value FROM paper_tags pt JOIN source_records r ON r.id=pt.source_record_id JOIN tags t ON t.id=pt.tag_id ORDER BY r.external_id').fetchall()
+            self.assertEqual(actual,[('2601.00001','cs.AI'),('2601.00002','cs.LG')])
+
+    def test_migrate_legacy_categories_without_removing_custom_evidence(self):
+        store=Store(self.db);self.addCleanup(store.conn.close)
+        paper,record=store.record('arxiv','2601.00001','https://arxiv.org/abs/2601.00001','A title','AB',[])
+        for namespace,key in [('arxiv:category','arxiv:categories'),('custom:topic','manual:1')]:
+            tag=store.conn.execute('INSERT INTO tags(namespace,value) VALUES(?,?)',(namespace,'cs.AI')).lastrowid
+            store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,key))
+        refresh_categories(store,paper,record,'cs.AI cs.LG');refresh_categories(store,paper,record,'cs.AI cs.LG')
+        self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_tags').fetchone()[0],3)
+        self.assertEqual(store.conn.execute("SELECT count(*) FROM paper_tags WHERE evidence_key='arxiv:categories'").fetchone()[0],0)
+        self.assertEqual(store.conn.execute("SELECT count(*) FROM paper_tags WHERE evidence_key='manual:1'").fetchone()[0],1)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_index_core.py

@@ -4,11 +4,22 @@ import tempfile
 import unittest
 from pathlib import Path
 
-from index_core import Store
-from import_records import ingest_file
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.records import ingest_file
 
 
 class StoreTests(unittest.TestCase):
+    def test_partial_source_refresh_preserves_known_abstract_and_can_later_restore_it(self):
+        paper,record=self.store.record('cvf','paper','https://example.org/paper','Known title','Known abstract',['Jane Doe'])
+        self.store.record('cvf','paper','https://example.org/paper','Known title',None,[],metadata={'abstract_fetch_error':{'status':404}})
+        self.store.record('cvf','paper','https://example.org/paper','Known title',None,[],metadata={'abstract_fetch_error':{'status':404}})
+        c=self.store.conn
+        self.assertEqual(c.execute('SELECT abstract,authors_json FROM papers').fetchone(),('Known abstract','["Jane Doe"]'))
+        source=c.execute('SELECT abstract,metadata_json FROM source_records WHERE id=?',(record,)).fetchone()
+        self.assertIsNone(source[0]);self.assertEqual(json.loads(source[1])['last_known_abstract']['abstract'],'Known abstract')
+        self.store.record('cvf','paper','https://example.org/paper','Updated title','Restored revised abstract',['Jane Doe'])
+        self.assertEqual(c.execute('SELECT title,abstract FROM papers').fetchone(),('Updated title','Restored revised abstract'))
+
     def setUp(self):
         self.tmp = tempfile.TemporaryDirectory()
         self.addCleanup(self.tmp.cleanup)
@@ -44,7 +55,7 @@ class StoreTests(unittest.TestCase):
 
     def test_manifest_count_mismatch_rolls_back_records(self):
         path=Path(self.tmp.name)/'source.jsonl'
-        path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'})+'\n')
+        path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'})+'\n')
         path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':2}))
         with self.assertRaisesRegex(ValueError,'differs'):
             ingest_file(self.db,path)
@@ -89,7 +100,7 @@ class StoreTests(unittest.TestCase):
 
     def test_missing_abstract_is_checked_even_when_manifest_claims_complete(self):
         path=Path(self.tmp.name)/'incomplete.jsonl'
-        row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':None,'authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'}
+        row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':None,'authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'}
         path.write_text(json.dumps(row)+'\n')
         path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':1,'status':'collected','missing_abstract_count':0}))
         ingest_file(self.db,path)
@@ -97,7 +108,7 @@ class StoreTests(unittest.TestCase):
 
     def test_manifest_edition_mismatch_rolls_back_instead_of_marking_coverage(self):
         path=Path(self.tmp.name)/'wrong-edition.jsonl'
-        row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'}
+        row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'}
         path.write_text(json.dumps(row)+'\n')
         path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':1,'year':2024}))
         with self.assertRaisesRegex(ValueError,'edition differs'):

tests/test_maintenance.py

@@ -3,8 +3,8 @@ import tempfile
 import unittest
 from unittest.mock import patch
 
-from index_core import Store
-from merge_conferences import merge
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.maintenance import merge
 
 
 class MaintenanceTests(unittest.TestCase):
@@ -17,14 +17,14 @@ class MaintenanceTests(unittest.TestCase):
     def test_completed_archive_is_not_started_again_after_maintenance(self):
         with tempfile.TemporaryDirectory() as tmp:
             db,archive=self.fixture(tmp,'complete')
-            with patch('merge_conferences.property_value',return_value='inactive'),patch('merge_conferences.subprocess.run') as run:
+            with patch('paper_atlas.ingestion.maintenance.property_value',return_value='inactive'),patch('paper_atlas.ingestion.maintenance.subprocess.run') as run:
                 merge(db,archive,[])
             run.assert_not_called()
 
     def test_inactive_incomplete_arxiv_requires_investigation(self):
         with tempfile.TemporaryDirectory() as tmp:
             db,archive=self.fixture(tmp,'failed')
-            with patch('merge_conferences.property_value',return_value='inactive'),patch('merge_conferences.subprocess.run') as run:
+            with patch('paper_atlas.ingestion.maintenance.property_value',return_value='inactive'),patch('paper_atlas.ingestion.maintenance.subprocess.run') as run:
                 with self.assertRaisesRegex(RuntimeError,'neither active nor verified complete'):
                     merge(db,archive,[])
             run.assert_not_called()
@@ -32,7 +32,7 @@ class MaintenanceTests(unittest.TestCase):
     def test_arxiv_finishing_at_the_stop_boundary_is_not_restarted(self):
         with tempfile.TemporaryDirectory() as tmp:
             db,archive=self.fixture(tmp,'complete')
-            with patch('merge_conferences.property_value',return_value='active'),patch('merge_conferences.subprocess.run') as run:
+            with patch('paper_atlas.ingestion.maintenance.property_value',return_value='active'),patch('paper_atlas.ingestion.maintenance.subprocess.run') as run:
                 merge(db,archive,[])
             self.assertEqual(run.call_args_list[0].args[0],['systemctl','stop','paper-atlas-arxiv-import.service'])
             self.assertEqual(run.call_count,1)

tests/test_mlsys_multimedia.py

@@ -0,0 +1,166 @@
+import hashlib
+import json
+from contextlib import closing
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors.proceedings import discover, parse_paper
+from paper_atlas.collectors.dblp import XML_SHA
+from paper_atlas.collectors.dblp_records import identifiers, normalize_row, collect
+from paper_atlas.collectors.dblp_scope import scope
+from paper_atlas.collectors.public_openreview import normalize
+from paper_atlas.db.submissions import identities_for_row
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.io.publication_ids import mlsys_id
+
+
+HASH = '0123456789abcdef' * 2
+ROOT = 'https://proceedings.mlsys.org/'
+
+
+def page(year=2026, modern=True):
+    return ROOT + ('paper_files/paper/' if modern else 'paper/') + str(year) + '/hash/' + HASH + ('-Abstract-Conference.html' if modern else '-Abstract.html')
+
+
+def raw_row(family='mm', book='ACM Multimedia', year=2024):
+    return {'record_type': 'inproceedings', 'key': f'conf/{family}/Paper24', 'publtype': None,
+            'fields': {'title': ['A published paper'], 'author': ['Jane Doe', 'John Smith'],
+                       'booktitle': [book], 'year': [str(year)], 'crossref': [f'conf/{family}/{year}']}}
+
+
+def volume(family='mm', year=2024, title=None):
+    return {'record_type': 'proceedings', 'key': f'conf/{family}/{year}',
+            'fields': {'title': [title or f'ACM Multimedia {year}'], 'year': [str(year)]}}
+
+
+class MlsysMultimediaTests(unittest.TestCase):
+    def test_mlsys_discovery_retains_all_official_years_and_ignores_pdf_or_other_host(self):
+        html = ''.join(f'<a href="/paper_files/paper/{year}">MLSys {year}</a>' for year in range(2019, 2027))
+        html += '<a href="https://example.org/paper_files/paper/2018">Other</a><a href="/paper/2026/file/paper.pdf">PDF</a>'
+        with patch('paper_atlas.collectors.proceedings.fetch', return_value=html.encode()):
+            self.assertEqual([year for _, year in discover('mlsys', '/unused')], list(range(2019, 2027)))
+
+    def test_mlsys_metadata_has_stable_publisher_identity_and_explicit_track(self):
+        html = '<meta name="citation_title" content="A title"><meta name="citation_author" content="Doe, Jane"><span class="paper-track">Conference</span><h2>Abstract</h2><p>Real abstract.</p><h4>Name Change Policy</h4><p>Footer.</p>'
+        row = parse_paper(html, page(), 'mlsys', 2026, ROOT + 'paper_files/paper/2026')
+        self.assertEqual((row['venue'], row['track'], row['abstract'], row['authors']), ('MLSys', 'main', 'Real abstract.', ['Jane Doe']))
+        self.assertEqual(row['identifiers']['publication_mlsys'], '2026/' + HASH)
+        with self.assertRaisesRegex(ValueError, 'Unclassified'):
+            parse_paper(html.replace('Conference', 'New Unknown Track'), page(), 'mlsys', 2026, ROOT)
+        with self.assertRaisesRegex(ValueError, 'publication year'):
+            parse_paper(html, page(), 'mlsys', 2025, ROOT)
+        with self.assertRaisesRegex(ValueError, 'non-archival'):
+            parse_paper(html, page(2018), 'mlsys', 2018, ROOT)
+
+    def test_year_label_and_retained_bibliography_links_bridge_the_2023_format(self):
+        url = page(2023).replace('-Conference.html', '-mlsys2023.html')
+        self.assertEqual(mlsys_id(url), '2023/' + HASH)
+        self.assertIsNone(mlsys_id(url.replace('mlsys2023.html', 'mlsys2022.html')))
+        official = parse_paper('<h1>Updated title</h1><span class="paper-track">mlsys2023</span><p class="paper-abstract">Official abstract</p>', url, 'mlsys', 2023, ROOT)
+        self.assertEqual(official['track'], 'main')
+        legacy = {'source': 'dblp', 'external_id': 'conf/mlsys/paper23', 'url': 'https://dblp.org/rec/conf/mlsys/paper23',
+                  'venue': 'MLSys', 'year': 2023, 'track': 'main', 'status': 'accepted', 'title': 'Bibliographic title',
+                  'abstract': None, 'authors': [], 'identifiers': {}, 'metadata': {'bibliography': {'ee': [url]}}}
+        self.assertEqual(identities_for_row(legacy), [('publication_mlsys', '2023/' + HASH)])
+        self.assertEqual(identities_for_row({**legacy, 'status': 'unknown'}), [])
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp); db = root / 'papers.sqlite'
+            for row in [legacy, official]:
+                path = root / (row['source'] + '.jsonl'); path.write_text(json.dumps(row) + '\n')
+                path.with_suffix('.manifest.json').write_text(json.dumps({'source': row['source'], 'parser_version': 2, 'list_url': ROOT, 'count': 1}))
+                ingest_file(db, path)
+            with closing(sqlite3.connect(db)) as c:
+                self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+                self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 1)
+                self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0], 2)
+
+    def test_legacy_mlsys_and_sysml_alias_preserve_same_publication_identity(self):
+        self.assertEqual(mlsys_id(page(2019, False)), mlsys_id(page(2019)))
+        self.assertIsNone(mlsys_id(page().replace('proceedings.mlsys.org', 'example.org')))
+        own, ambiguous = identifiers({'ee': [page(2019, False), page(2019)], 'cite': [page(2025)]})
+        self.assertEqual(own, {'publication_mlsys': '2019/' + HASH}); self.assertFalse(ambiguous)
+        _, ambiguous = identifiers({'ee': [page(2019), page(2019).replace(HASH, 'a' * 32)]})
+        self.assertEqual(len(ambiguous['publication_mlsys']), 2)
+        row = parse_paper('<h1>Legacy</h1><p class="paper-abstract">Abstract</p>', page(2019, False), 'mlsys', 2019, ROOT)
+        self.assertEqual(row['venue_aliases'], ['SysML'])
+        self.assertEqual(row['track'], 'main')
+
+    def test_mm_main_numbered_parts_and_workshops_remain_separate(self):
+        for book in ['ACM Multimedia', 'ACM Multimedia (1)', 'ACM Multimedia (2)']:
+            result, reason = scope(raw_row(book=book), volume())
+            self.assertIsNone(reason); self.assertEqual(result['venue'], 'MM'); self.assertEqual(result['track'], 'proceedings_unspecified')
+            self.assertIn('per-paper section', result['track_scope_note'])
+            self.assertIn('ACMMM', result['venue_aliases'])
+        self.assertEqual(scope(raw_row(book='ACM Multimedia Workshops'), volume())[0]['track'], 'workshop')
+        self.assertEqual(scope(raw_row(book='ACM Multimedia (Thematic Workshops)'), volume())[0]['track'], 'workshop')
+        self.assertIsNone(scope(raw_row(book='MMSports@MM'), volume())[0])
+        self.assertEqual(normalize_row(raw_row(book='ACM Multimedia Workshops'), {'conf/mm/2024': volume()}, {'MM'})[1], 'separate_workshop')
+
+    def test_sysml_alias_requires_machine_learning_proceedings(self):
+        row = raw_row('mlsys', 'SysML', 2019)
+        result, reason = scope(row, volume('mlsys', 2019, 'Conference on Machine Learning and Systems, SysML 2019'))
+        self.assertIsNone(reason); self.assertEqual(result['venue'], 'MLSys'); self.assertEqual(result['venue_aliases'], ['SysML'])
+        self.assertIsNone(scope(row, volume('mlsys', 2019, 'Systems Modeling Language 2019'))[0])
+
+    def test_bibliography_and_official_abstract_import_as_one_paper_and_submission(self):
+        raw = raw_row('mlsys', 'SysML', 2019); raw['fields']['ee'] = [page(2019, False)]
+        bib, reason = normalize_row(raw, {'conf/mlsys/2019': volume('mlsys', 2019, 'Machine Learning and Systems 2019')}, {'MLSys'})
+        self.assertIsNone(reason)
+        official = parse_paper('<meta name="citation_title" content="An updated title"><p class="paper-abstract">The official abstract.</p>', page(2019), 'mlsys', 2019, ROOT)
+        # Different title and absent official authors deliberately prevent a title-based bridge.
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp); db = root / 'papers.sqlite'
+            for row in [bib, official]:
+                path = root / (row['source'] + '.jsonl'); path.write_text(json.dumps(row) + '\n')
+                path.with_suffix('.manifest.json').write_text(json.dumps({'source': row['source'], 'parser_version': 2,
+                    'list_url': ROOT, 'count': 1, 'venue': 'MLSys', 'year': 2019, 'track': 'main'}))
+                ingest_file(db, path)
+            with closing(sqlite3.connect(db)) as c:
+                self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+                self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 1)
+                self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0], 2)
+                self.assertEqual(c.execute('SELECT abstract FROM papers').fetchone()[0], 'The official abstract.')
+                self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
+
+    def test_publisher_identity_cannot_bridge_rejection_other_year_or_venue(self):
+        row = {'source': 'mlsys', 'venue': 'MLSys', 'year': 2026, 'status': 'accepted',
+               'identifiers': {'publication_mlsys': '2026/' + HASH}}
+        self.assertEqual(identities_for_row(row), [('publication_mlsys', '2026/' + HASH)])
+        self.assertEqual(identities_for_row({**row, 'status': 'rejected'}), [])
+        with self.assertRaisesRegex(ValueError, 'Invalid authoritative'):
+            identities_for_row({**row, 'identifiers': {}})
+        for changed in [dict(year=2025), dict(venue='MM'), dict(source='curated')]:
+            bad = {**row, **changed, 'submission_identifiers': [{'scheme': 'publication_mlsys', 'value': '2026/' + HASH}]}
+            with self.assertRaisesRegex(ValueError, 'Invalid authoritative'):
+                identities_for_row(bad)
+
+    def test_explicit_mm_negative_assertion_maps_to_mm_without_inventing_labels(self):
+        provenance = {'source_file': 'https://example.org/pinned', 'source_line': 1}
+        row = {'id': 'realForum1', 'title': 'A title', 'abstract': 'An abstract', 'track': 'main', 'status': 'Reject'}
+        normalized, reason = normalize(row, 'acmmm', 2025, provenance)
+        self.assertIsNone(reason); self.assertEqual(normalized['venue'], 'MM')
+        self.assertEqual(normalized['status'], 'rejected')
+        self.assertIsNone(normalize({**row, 'status': 'Submitted'}, 'acmmm', 2025, provenance)[0])
+
+    def test_old_candidate_subset_cannot_silently_omit_new_requested_venues(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp); catalogue = root / 'dblp-catalogue.sqlite'
+            with closing(sqlite3.connect(catalogue)) as c, c:
+                c.execute('CREATE TABLE proceedings(key TEXT PRIMARY KEY,metadata_json TEXT)')
+                c.execute('INSERT INTO proceedings VALUES(?,?)', ('conf/mm/2024', json.dumps(volume())))
+            candidates = root / 'candidates.jsonl'; candidates.write_text(json.dumps(raw_row()) + '\n')
+            original = candidates.read_bytes()
+            catalogue.with_name('dblp-ai-candidates.manifest.json').write_text(json.dumps({
+                'snapshot_sha256': XML_SHA, 'candidate_sha256': hashlib.sha256(original).hexdigest(),
+                'candidate_family_counts': {'aaai': 1}, 'record_types': {'proceedings': 1}}))
+            with self.assertRaisesRegex(ValueError, 'original --archive'):
+                collect(catalogue, root / 'out', ['MM'], candidates=candidates)
+            self.assertEqual(candidates.read_bytes(), original)
+            self.assertFalse((root / 'out').exists())
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_openreview_snapshot.py

@@ -7,8 +7,8 @@ import tarfile
 import tempfile
 import unittest
 
-from collect_openreview_snapshot import collect,decision_record,paper_record
-from import_records import ingest_file
+from paper_atlas.collectors.openreview import collect,decision_record,paper_record
+from paper_atlas.ingestion.records import ingest_file
 
 
 class SnapshotTests(unittest.TestCase):

tests/test_peerread.py

@@ -5,9 +5,9 @@ import unittest
 from pathlib import Path
 from unittest.mock import patch
 
-from collect_peerread import collect
-from import_records import ingest_file
-from index_core import Store
+from paper_atlas.collectors.peerread import collect
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.db.store import Store
 
 
 class PeerReadTests(unittest.TestCase):
@@ -16,7 +16,7 @@ class PeerReadTests(unittest.TestCase):
         paper={'conference':'ICLR 2017 conference submission','id':'123','title':'Public paper','abstract':'Abstract','authors':'Jane Doe, John Smith','accepted':False,'reviews':[{'text':'Do not persist this review body'}]}
         with tempfile.TemporaryDirectory() as tmp:
             root=Path(tmp)
-            with patch('collect_peerread.fetch',side_effect=[json.dumps({'sha':'a'*40}).encode(),json.dumps(tree).encode()]),patch('collect_peerread.urllib.request.urlopen',return_value=io.BytesIO(json.dumps(paper).encode())),patch('collect_peerread.time.sleep'):
+            with patch('paper_atlas.collectors.peerread.fetch',side_effect=[json.dumps({'sha':'a'*40}).encode(),json.dumps(tree).encode()]),patch('paper_atlas.collectors.peerread.urllib.request.urlopen',return_value=io.BytesIO(json.dumps(paper).encode())),patch('paper_atlas.collectors.peerread.time.sleep'):
                 collect(root/'cache',root/'out')
             path=root/'out/peerread-iclr2017-main.jsonl'
             record=json.loads(path.read_text())

tests/test_pmlr.py

@@ -4,7 +4,7 @@ import json
 from pathlib import Path
 from unittest.mock import patch
 
-from collect_pmlr import load_metadata,venue_key,collect_volume
+from paper_atlas.collectors.pmlr import load_metadata,venue_key,collect_volume
 
 
 class MetadataTests(unittest.TestCase):
@@ -25,9 +25,9 @@ class MetadataTests(unittest.TestCase):
         self.assertEqual(venue_key('ECML-PKDD'),'ECML-PKDD')
 
     def test_missing_abstract_is_reported_as_partial_collection(self):
-        config=b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n'
+        config=b'shortname: ICML\nyear: 2023\nbooktitle: ICML\n'
         paper='---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nabstract:\n---\n'
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter([('paper.md',paper)])):
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.pmlr.fetch',return_value=config),patch('paper_atlas.collectors.pmlr.metadata_texts',return_value=iter([('paper.md',paper)])):
             result=collect_volume(999,tmp,tmp)
         self.assertEqual(result['count'],1)
         self.assertEqual(result['missing_abstract_count'],1)
@@ -37,7 +37,7 @@ class MetadataTests(unittest.TestCase):
         config=b'shortname: CoRL\nyear: 2023\nbooktitle: CoRL\n'
         paper='---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nabstract: Real abstract\ndate: {date}\n---\n'
         posts=[('2023-11-13-paper1.md',paper.format(date='2023-11-13')),('2023-12-02-paper1.md',paper.format(date='2023-12-02'))]
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter(posts)):
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.pmlr.fetch',return_value=config),patch('paper_atlas.collectors.pmlr.metadata_texts',return_value=iter(posts)):
             result=collect_volume(229,tmp,tmp)
             rows=[json.loads(line) for line in Path(result['file']).read_text().splitlines()]
         self.assertEqual(result['count'],1)
@@ -49,7 +49,7 @@ class MetadataTests(unittest.TestCase):
     def test_conflicting_duplicate_posts_do_not_replace_previous_collection(self):
         config=b'shortname: CoRL\nyear: 2023\nbooktitle: CoRL\n'
         paper='---\nid: paper1\nlayout: inproceedings\ntitle: {title}\nabstract: Abstract\n---\n'
-        with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter([('a.md',paper.format(title='First paper')),('b.md',paper.format(title='Other paper'))])):
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.pmlr.fetch',return_value=config),patch('paper_atlas.collectors.pmlr.metadata_texts',return_value=iter([('a.md',paper.format(title='First paper')),('b.md',paper.format(title='Other paper'))])):
             previous=Path(tmp)/'pmlr-v229.jsonl';previous.write_text('previous valid collection')
             with self.assertRaisesRegex(ValueError,'Conflicting metadata posts'):
                 collect_volume(229,tmp,tmp)

tests/test_pmlr_track_migration.py

@@ -0,0 +1,156 @@
+import copy
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors.pmlr_tracks import program_tracks, record_track
+from paper_atlas.db.store import Store
+from paper_atlas.db.submissions import identities_for_row
+from paper_atlas.ingestion.pmlr_tracks import reclassify, plan
+from paper_atlas.ingestion.records import ingest_file
+from tests.test_pmlr_tracks import event, program, record
+
+
+class PmlrMigrationTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name); self.db = self.root / 'papers.sqlite'
+        self.inventory = self.root / 'inventory.json'
+        self.rows = [record(title='A main research paper'),
+                     record('position1', 'v267/paper2', title='A position paper')]
+        self.proof = {'url': 'https://icml.cc/static/virtual/data/icml-2025-orals-posters.json', 'sha256': 'a' * 64}
+        mapping, _ = program_tracks(program(event(), event('position1', 'Position_Paper_Track', 2)), 2025)
+        store = Store(self.db); c = store.conn
+        for row in self.rows:
+            row.update(url='https://proceedings.mlr.press/' + row['external_id'] + '.html',
+                       list_url='https://proceedings.mlr.press/v267/', status='accepted')
+            paper, source = store.record('pmlr', row['external_id'], row['url'], row['title'], row['abstract'],
+                                         row['authors'], row['identifiers'], {'openreview': row['identifiers']['openreview']})
+            store.decision(paper, source, 'ICML', 2025, 'main', row['external_id'], row['url'],
+                           'accepted', 'listed in official proceedings', identities=identities_for_row(row))
+            record_track(row, mapping, self.proof)
+        c.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','keep me')")
+        c.execute("INSERT INTO paper_tags VALUES(1,1,NULL,'custom:manual')")
+        c.commit(); c.close()
+        self.write_inventory()
+
+    def write_inventory(self):
+        files = []
+        for row in self.rows:
+            path = self.root / (row['track'] + '.jsonl')
+            data = (json.dumps(row) + '\n').encode(); path.write_bytes(data)
+            m = {'source': 'pmlr', 'venue': 'ICML', 'year': 2025, 'volume': 267,
+                 'parser_version': 3, 'track': row['track'], 'track_program': self.proof,
+                 'count': 1, 'file': str(path), 'file_sha256': hashlib.sha256(data).hexdigest(),
+                 'missing_abstract_count': 0, 'list_url': row['list_url'], 'status': 'collected'}
+            path.with_suffix('.manifest.json').write_text(json.dumps(m)); files.append(m)
+        self.inventory.write_text(json.dumps({'source': 'pmlr', 'venue': 'ICML', 'year': 2025,
+                                              'volume': 267, 'parser_version': 3, 'track_program': self.proof,
+                                              'count': 2, 'files': files}))
+
+    def tracks(self):
+        with sqlite3.connect(self.db) as c:
+            return c.execute('SELECT s.id,e.track FROM submissions s JOIN editions e ON e.id=s.edition_id ORDER BY s.id').fetchall()
+
+    def test_plan_is_read_only_and_reports_exact_original_event(self):
+        result = reclassify(self.db, [self.inventory], self.root / 'plan.json')
+        self.assertEqual((result['status'], result['records_verified'], result['events_to_move']), ('planned', 2, 1))
+        self.assertEqual(result['changes'][0]['submission_id'], 2)
+        self.assertEqual(self.tracks(), [(1, 'main'), (2, 'main')])
+
+    def test_apply_preserves_ids_decisions_text_tags_and_is_idempotent(self):
+        with sqlite3.connect(self.db) as c:
+            before = {table: c.execute('SELECT * FROM ' + table).fetchall() for table in ('papers', 'decisions', 'decision_sources', 'submission_sources', 'paper_tags')}
+        result = reclassify(self.db, [self.inventory], self.root / 'apply.json', True)
+        self.assertEqual((result['status'], result['events_moved']), ('complete', 1))
+        self.assertEqual(self.tracks(), [(1, 'main'), (2, 'position')])
+        with sqlite3.connect(self.db) as c:
+            for table, rows in before.items():
+                self.assertEqual(c.execute('SELECT * FROM ' + table).fetchall(), rows, table)
+            self.assertEqual(c.execute('SELECT count(*) FROM submission_identifiers i JOIN submissions s ON s.id=i.submission_id WHERE i.edition_id<>s.edition_id').fetchone()[0], 0)
+            self.assertEqual(c.execute('SELECT e.track,es.imported_count FROM edition_sources es JOIN editions e ON e.id=es.edition_id ORDER BY e.track').fetchall(), [('main', 1), ('position', 1)])
+            self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
+        again = reclassify(self.db, [self.inventory], self.root / 'again.json', True)
+        self.assertEqual(again['events_moved'], 0)
+        for m in json.loads(self.inventory.read_text())['files']:
+            ingest_file(self.db, m['file'])
+        self.assertEqual(self.tracks(), [(1, 'main'), (2, 'position')])
+        with sqlite3.connect(self.db) as c:
+            self.assertEqual(c.execute('SELECT * FROM decisions').fetchall(), before['decisions'])
+
+    def test_conflicting_target_event_stops_without_merging_history(self):
+        store = Store(self.db)
+        _, source = store.record('other', 'position1', 'https://example.org', 'Other evidence', None, [], {'openreview': 'position1'})
+        store.decision(2, source, 'ICML', 2025, 'position', 'position1', 'https://example.org', 'rejected', 'Reject', identities=[('openreview', 'position1')])
+        store.conn.commit(); store.conn.close()
+        before = self.tracks()
+        with self.assertRaisesRegex(ValueError, 'another event'):
+            reclassify(self.db, [self.inventory], self.root / 'blocked.json', True)
+        self.assertEqual(self.tracks(), before)
+
+    def test_shared_event_requires_reconciliation_and_keeps_all_sources(self):
+        store = Store(self.db)
+        _, source = store.record('mirror', 'position1', 'https://example.org', 'Mirror title', None, [], {'openreview': 'position1'})
+        store.conn.execute('INSERT INTO submission_sources VALUES(?,?,?)', (2, source, 'position1'))
+        store.conn.commit(); store.conn.close()
+        with self.assertRaisesRegex(ValueError, 'multiple sources'):
+            reclassify(self.db, [self.inventory], self.root / 'shared.json', True)
+        self.assertEqual(self.tracks(), [(1, 'main'), (2, 'main')])
+
+    def test_text_mismatch_in_last_record_prevents_any_changes(self):
+        with sqlite3.connect(self.db) as c:
+            c.execute("UPDATE source_records SET abstract='Changed elsewhere' WHERE id=2")
+        with self.assertRaisesRegex(ValueError, 'preserve existing publisher text'):
+            reclassify(self.db, [self.inventory], self.root / 'text.json', True)
+        self.assertEqual(self.tracks(), [(1, 'main'), (2, 'main')])
+
+    def test_mutation_failure_rolls_back_editions_metadata_and_identifiers(self):
+        calls = 0
+        def fail_after_updates(c, scopes):
+            nonlocal calls
+            calls += 1
+            if calls == 2:
+                raise ValueError('Injected post-migration failure')
+            return plan(c, scopes)
+        with sqlite3.connect(self.db) as c:
+            before = c.execute('SELECT * FROM source_records').fetchall()
+        with patch('paper_atlas.ingestion.pmlr_tracks.plan', side_effect=fail_after_updates):
+            with self.assertRaisesRegex(ValueError, 'Injected'):
+                reclassify(self.db, [self.inventory], self.root / 'rollback.json', True)
+        self.assertEqual(self.tracks(), [(1, 'main'), (2, 'main')])
+        with sqlite3.connect(self.db) as c:
+            self.assertEqual(c.execute('SELECT * FROM source_records').fetchall(), before)
+            self.assertEqual(c.execute("SELECT count(*) FROM editions WHERE track='position'").fetchone()[0], 0)
+        self.assertEqual(json.loads((self.root / 'rollback.json').read_text())['status'], 'error')
+
+    def test_changed_evidence_and_active_writer_are_refused(self):
+        out = self.root / 'blocked.json'
+        with Path(str(self.db) + '.maintenance.lock').open('a') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            with self.assertRaises(BlockingIOError):
+                reclassify(self.db, [self.inventory], out, True)
+        self.assertFalse(out.exists())
+        path = self.root / 'position.jsonl'; path.write_text(path.read_text() + ' ')
+        with self.assertRaisesRegex(ValueError, 'differs from its verified inventory'):
+            reclassify(self.db, [self.inventory], out, True)
+        self.assertEqual(self.tracks(), [(1, 'main'), (2, 'main')])
+
+    def test_input_report_and_duplicate_forums_are_refused(self):
+        original = self.inventory.read_bytes()
+        with self.assertRaisesRegex(ValueError, 'preserve evidence'):
+            reclassify(self.db, [self.inventory], self.inventory, True)
+        self.assertEqual(self.inventory.read_bytes(), original)
+        self.rows[1]['identifiers']['openreview'] = self.rows[0]['identifiers']['openreview']
+        self.rows[1]['metadata']['track_evidence']['identity_value'] = self.rows[0]['identifiers']['openreview']
+        self.write_inventory()
+        with self.assertRaisesRegex(ValueError, 'repeated forum'):
+            reclassify(self.db, [self.inventory], self.root / 'duplicate.json', True)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_pmlr_tracks.py

@@ -0,0 +1,148 @@
+import copy
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors.pmlr import collect_volume
+from paper_atlas.collectors.pmlr_tracks import program_tracks, record_track, position_list_policy
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.ingestion.backfill import backfill
+
+
+def event(forum='realForum1', track='Conference', event_id=1, year=2025, **values):
+    return {'id': event_id, 'sourceid': event_id, 'name': 'A published paper',
+            'sourceurl': f'https://openreview.net/group?id=ICML.cc/{year}/' + track,
+            'paper_url': 'https://openreview.net/forum?id=' + forum,
+            'decision': 'Accept (Poster)', **values}
+
+
+def program(*events, **changes):
+    return json.dumps({'count': len(events), 'results': list(events), **changes}).encode()
+
+
+def record(forum='realForum1', external='v267/paper1', year=2025, title='A published paper'):
+    return {'source': 'pmlr', 'external_id': external, 'venue': 'ICML', 'year': year,
+            'title': title, 'abstract': 'Original abstract', 'authors': ['Jane Doe', 'John Smith'],
+            'identifiers': {'openreview': forum}, 'metadata': {}, 'track': 'main'}
+
+
+class PmlrTrackTests(unittest.TestCase):
+    def provenance(self):
+        return {'url': 'https://icml.cc/static/virtual/data/icml-2025-orals-posters.json', 'sha256': 'a' * 64}
+
+    def test_real_groups_not_title_prefixes_determine_modern_tracks(self):
+        mapping, inventory = program_tracks(program(
+            event(name='Position: still a main-track paper'),
+            event('position1', 'Position_Paper_Track', 2, name='No prefix needed'),
+            event('2025-Oral-fake', event_id=3, sourceid=-1),
+            event('workshop1', 'Workshop', 4)), 2025)
+        main = record(title='Position: still a main-track paper')
+        position = record('position1', 'v267/paper2', title='No prefix needed')
+        record_track(main, mapping, self.provenance()); record_track(position, mapping, self.provenance())
+        self.assertEqual((main['track'], position['track']), ('main', 'position'))
+        self.assertEqual(inventory['paper_identity_count'], 2)
+        self.assertEqual(inventory['excluded_events'], {'synthetic_oral_event': 1, 'other_source': 1})
+
+    def test_conflicting_forums_pagination_and_missing_real_ids_fail(self):
+        for data in [program(event(), event(track='Position_Paper_Track', event_id=2)),
+                     program(event(), next='page2'), program(event(), count=2),
+                     program(event(paper_url='')), program(event(decision='Reject')),
+                     program(event(), event())]:
+            with self.subTest(data=data), self.assertRaises(ValueError):
+                program_tracks(data, 2025)
+
+    def test_missing_publication_identity_has_no_main_track_fallback(self):
+        mapping, _ = program_tracks(program(event()), 2025)
+        with self.assertRaisesRegex(ValueError, 'absent'):
+            record_track(record('unknownForum'), mapping, self.provenance())
+
+    def test_2024_requires_organizer_endorsed_list_and_publisher_self_links(self):
+        html = b'<a href="/virtual/2024/papers.html?filter=titles&amp;search=position%3A">position papers</a>'
+        policy = position_list_policy(html)
+        with self.assertRaises(ValueError):
+            position_list_policy(html.replace(b'2024', b'2023'))
+        e = event(year=2024, name='Position: An explicit position', paper_url='',
+                  paper_pdf_url='https://proceedings.mlr.press/v235/paper1.html')
+        with self.assertRaisesRegex(ValueError, 'policy'):
+            program_tracks(program(e), 2024)
+        mapping, _ = program_tracks(program(e), 2024, policy)
+        row = record(external='v235/paper1', year=2024)
+        record_track(row, mapping, {**self.provenance(), 'position_list_policy': policy})
+        self.assertEqual(row['track'], 'position')
+        self.assertEqual(row['metadata']['track_evidence']['identity_scheme'], 'pmlr')
+
+    def test_reviewed_2024_link_alias_still_needs_title_and_two_authors(self):
+        e = event(year=2024, paper_url='', paper_pdf_url='https://proceedings.mlr.press/v235/theo24a.html',
+                  authors=[{'fullname': 'Jane Doe'}, {'fullname': 'John Smith'}])
+        mapping, _ = program_tracks(program(e), 2024, {'verified': True})
+        row = record(external='v235/guyard24a', year=2024)
+        record_track(row, mapping, self.provenance())
+        self.assertEqual(row['metadata']['track_evidence']['corroborating_author_count'], 2)
+        self.assertNotIn('_authors', row['metadata']['track_evidence'])
+        for changed in [dict(title='Different title'), dict(authors=['Someone Else'])]:
+            bad = record(external='v235/guyard24a', year=2024); bad.update(changed)
+            with self.assertRaisesRegex(ValueError, 'corroboration'):
+                record_track(bad, mapping, self.provenance())
+
+    def test_collector_splits_tracks_preserves_ids_and_imports_one_event_each(self):
+        config = b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n'
+        template = '---\nid: {id}\nlayout: inproceedings\ntitle: {title}\nabstract: Original abstract\nopenreview: {forum}\n---\n'
+        posts = [('a.md', template.format(id='paper1', title='Main paper', forum='realForum1')),
+                 ('b.md', template.format(id='paper2', title='Position paper', forum='position1'))]
+        mapping, inventory = program_tracks(program(event(), event('position1', 'Position_Paper_Track', 2)), 2025)
+        with tempfile.TemporaryDirectory() as tmp, patch('paper_atlas.collectors.pmlr.fetch', return_value=config), \
+                patch('paper_atlas.collectors.pmlr.metadata_texts', return_value=iter(posts)), \
+                patch('paper_atlas.collectors.pmlr.collect_track_evidence', return_value=(mapping, self.provenance())):
+            result = collect_volume(267, tmp, tmp)
+            self.assertNotIn('file', result)
+            self.assertEqual(result['track'], 'mixed')
+            self.assertEqual({m['track']: m['count'] for m in result['files']}, {'main': 1, 'position': 1})
+            self.assertEqual(result['count'], 2)
+            db = Path(tmp) / 'papers.sqlite'
+            for manifest in result['files']:
+                ingest_file(db, manifest['file'])
+            import sqlite3
+            with sqlite3.connect(db) as c:
+                self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 2)
+                self.assertEqual(c.execute('SELECT track,count(*) FROM submissions s JOIN editions e ON s.edition_id=e.id GROUP BY track').fetchall(), [('main', 1), ('position', 1)])
+                self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
+
+    def test_unknown_track_does_not_replace_existing_volume_output(self):
+        config = b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n'
+        post = '---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nopenreview: missing123\n---\n'
+        with tempfile.TemporaryDirectory() as tmp, patch('paper_atlas.collectors.pmlr.fetch', return_value=config), \
+                patch('paper_atlas.collectors.pmlr.metadata_texts', return_value=iter([('a.md', post)])), \
+                patch('paper_atlas.collectors.pmlr.collect_track_evidence', return_value=({}, self.provenance())):
+            old = Path(tmp) / 'pmlr-v267.jsonl'; old.write_text('previous collection')
+            with self.assertRaisesRegex(ValueError, 'absent'):
+                collect_volume(267, tmp, tmp)
+            self.assertEqual(old.read_text(), 'previous collection')
+            self.assertFalse((Path(tmp) / 'pmlr-v267.inventory.json').exists())
+
+    def test_old_mixed_manifest_is_rejected_before_opening_database(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / 'pmlr-v267.jsonl'; path.write_text('')
+            path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'pmlr', 'venue': 'ICML', 'year': 2025, 'parser_version': 2}))
+            db = Path(tmp) / 'absent.sqlite'
+            for action in [lambda: ingest_file(db, path), lambda: backfill(db, [path])]:
+                with self.assertRaisesRegex(ValueError, 'Recollect mixed'):
+                    action()
+            self.assertFalse(db.exists())
+
+    def test_changed_track_or_own_identity_rolls_back_import(self):
+        from paper_atlas.collectors.pmlr_tracks import require_track_record
+        row = record(); mapping, _ = program_tracks(program(event()), 2025)
+        record_track(row, mapping, self.provenance())
+        manifest = {'source': 'pmlr', 'venue': 'ICML', 'year': 2025, 'parser_version': 3,
+                    'track_program': self.provenance()}
+        require_track_record(row, manifest)
+        for change in [dict(track='position'), dict(identifiers={'openreview': 'otherForum'})]:
+            bad = copy.deepcopy(row); bad.update(change)
+            with self.assertRaises(ValueError):
+                require_track_record(bad, manifest)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_postgres.py

@@ -0,0 +1,309 @@
+import contextlib
+from contextlib import closing
+import io
+import itertools
+import json
+import os
+from pathlib import Path
+import sqlite3
+import tempfile
+import unittest
+from unittest.mock import patch
+import uuid
+
+from paper_atlas.db.store import Store
+from paper_atlas.postgres import schema, text
+from paper_atlas.postgres.client import ReadConnection, Row, connect
+from paper_atlas.postgres.migration import migrate
+from paper_atlas import querying
+
+
+class SchemaTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temporary.cleanup)
+        self.path = Path(self.temporary.name) / 'source.sqlite'
+        self.store = Store(self.path)
+        self.addCleanup(self.store.conn.close)
+
+    def test_every_business_table_is_included_and_engine_tables_are_excluded(self):
+        plan = schema.inspect(self.store.conn)
+        self.assertEqual(set(plan), set(schema.TABLES))
+        self.assertEqual(len(plan), 21)
+        self.assertNotIn('papers_fts', plan)
+        self.assertEqual(plan['paper_tags']['pk'], ['paper_id', 'tag_id', 'evidence_key'])
+        self.assertEqual(len(plan['source_records']['foreign_keys']), 2)
+        sql = schema.table_sql('stage', 'papers', plan['papers'])
+        self.assertIn('GENERATED BY DEFAULT AS IDENTITY', sql)
+        self.assertIn('TEXT COLLATE "C"', sql)
+
+    def test_unknown_tables_and_check_constraints_fail_before_copy(self):
+        self.store.conn.execute('CREATE TABLE extra(id INTEGER PRIMARY KEY)')
+        with self.assertRaisesRegex(ValueError, 'Unexpected business'):
+            schema.inspect(self.store.conn)
+        self.store.conn.execute('DROP TABLE extra')
+        self.store.conn.execute('ALTER TABLE papers ADD COLUMN extra TEXT CHECK(extra IS NULL)')
+        with self.assertRaisesRegex(ValueError, 'CHECK'):
+            schema.inspect(self.store.conn)
+
+    def test_cutover_stops_legacy_writers_but_keeps_read_access(self):
+        self.store.conn.commit(); self.store.conn.close()
+        Path(str(self.path) + '.postgresql-cutover.json').write_text('{"status":"active"}')
+        with self.assertRaisesRegex(RuntimeError, 'cut over'):
+            Store(self.path)
+        with sqlite3.connect(self.path.as_uri() + '?mode=ro', uri=True) as c:
+            self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 0)
+
+    def test_rows_support_mapping_and_positional_query_access(self):
+        row = Row(['id', 'title'], (7, 'A'))
+        self.assertEqual(row[0], row['id'])
+        self.assertEqual(dict(row), {'id': 7, 'title': 'A'})
+
+    def test_connection_services_cannot_be_password_urls(self):
+        with self.assertRaisesRegex(ValueError, 'service name'):
+            connect('postgresql://user:secret@host/db')
+
+    def test_checks_compare_literal_values_not_only_constraint_counts(self):
+        clauses = schema.checks_in("CREATE TABLE t(x TEXT CHECK(x IN ('a','b)')),y TEXT CHECK(y IS NULL))")
+        self.assertEqual(clauses, ["x IN ('a','b)')", 'y IS NULL'])
+        self.assertEqual(schema.normalize_check("x IN ( 'a' )"), schema.normalize_check("x in('a')"))
+        self.assertNotEqual(schema.normalize_check("x IN ('A')"), schema.normalize_check("x IN ('a')"))
+
+    def test_text_codec_is_bijective_and_preserves_key_order_and_prefixes(self):
+        alphabet = ['\x00', '\x01', '\x02', 'a', 'é', '🙂']
+        values = [''.join(v) for n in range(4) for v in itertools.product(alphabet, repeat=n)]
+        for value in values:
+            encoded = text.encode(value)
+            self.assertNotIn('\x00', encoded)
+            self.assertEqual(text.decode(encoded), value)
+        self.assertEqual(sorted(map(text.encode, values)), list(map(text.encode, sorted(values))))
+        for value in [None, 7, 0.5]:
+            self.assertEqual(text.encode(value), value)
+            self.assertEqual(text.decode(value), value)
+        for value in ['\x01', 'prefix\x01x']:
+            with self.assertRaisesRegex(ValueError, 'Invalid'):
+                text.decode(value)
+
+
+@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires an explicitly configured remote PostgreSQL test service')
+class PostgreSQLIntegrationTests(unittest.TestCase):
+    def setUp(self):
+        self.service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
+        self.target = 'pa_test_' + uuid.uuid4().hex[:14]
+        self.temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temporary.cleanup)
+        self.db = Path(self.temporary.name) / 'source.sqlite'
+        self.out = Path(self.temporary.name) / 'report.json'
+        store = Store(self.db)
+        paper, record = store.record('arxiv', '2401.00001', 'https://arxiv.org/abs/2401.00001',
+            'Quiet graph models — 研究', 'Graph models learn reinforcement policies.\nQuoted "text", tabs\tand Unicode α.',
+            ['Jane Doe', 'John Smith'], {'arxiv': '2401.00001'}, {'nested': {'a': ['α', None, 1]}, 'raw': '\\n'})
+        store.record_tags(paper, record, 'arxiv', [{'namespace': 'arxiv:category', 'value': 'cs.AI'}])
+        store.decision(paper, record, 'ICLR', 2024, 'main', 'forum-reject', 'https://example.org/reject',
+                       'rejected', 'Reject', identities=[{'scheme': 'openreview', 'value': 'forum-reject'}], observed_at='2024-01-01T00:00:00Z')
+        same, second = store.record('official', '2025/paper', 'https://example.org/accepted',
+            'Quiet graph models — 研究', 'Graph models learn reinforcement policies.', ['Jane Doe', 'John Smith'], {'arxiv': '2401.00001'})
+        self.assertEqual(same, paper)
+        store.decision(paper, second, 'COLM', 2025, 'main', 'forum-accept', 'https://example.org/accept',
+                       'accepted', 'Accept', ['Conference on Language Modeling'], observed_at='2025-01-01T00:00:00Z')
+        store.record('arxiv', '2401.00002', 'https://arxiv.org/abs/2401.00002', 'Empty abstract', None, [])
+        tag = store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','manual')").lastrowid
+        store.conn.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)', (paper, tag, 'manual:one'))
+        store.conn.execute("INSERT INTO paper_merges(former_paper_id,surviving_paper_id,reason) VALUES(900,?,'fixture')", (paper,))
+        store.conn.execute("INSERT INTO submission_merges VALUES(5000,1,'fixture',CURRENT_TIMESTAMP)")
+        store.conn.execute("INSERT INTO decision_merges VALUES(6000,1,'fixture',CURRENT_TIMESTAMP)")
+        store.conn.execute("UPDATE sqlite_sequence SET seq=1000 WHERE name='papers'")
+        store.conn.commit(); store.conn.close()
+        self.addCleanup(self.cleanup_schemas)
+
+    def cleanup_schemas(self):
+        with connect(self.service) as pg:
+            names = pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (self.target,)).fetchall()
+            for name, in names:
+                pg.execute('DROP SCHEMA ' + schema.quote(name) + ' CASCADE')
+
+    def migrate(self):
+        with contextlib.redirect_stdout(io.StringIO()):
+            return migrate(self.db, self.service, self.out, schema=self.target, min_free_gib=0)
+
+    def published(self):
+        with connect(self.service) as pg:
+            return bool(pg.execute('SELECT 1 FROM pg_namespace WHERE nspname=%s', (self.target,)).fetchone())
+
+    def test_full_content_history_tags_sequences_and_readonly_queries(self):
+        report = self.migrate()
+        self.assertEqual(report['status'], 'complete')
+        self.assertEqual(len(report['tables']), 21)
+        for entry in report['tables'].values():
+            self.assertEqual(entry['source_rows'], entry['target_rows'])
+            self.assertEqual(entry['source_sha256'], entry['target_sha256'])
+        with connect(self.service) as pg, pg.transaction(force_rollback=True):
+            prefix = schema.quote(self.target) + '.'
+            value = pg.execute('INSERT INTO ' + prefix + 'papers(title,title_key) VALUES(%s,%s) RETURNING id', ('new', 'new')).fetchone()[0]
+            self.assertGreater(value, 1000)
+            record = pg.execute('INSERT INTO ' + prefix + "source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) SELECT source_id,'new',%s,url,title,abstract,authors_json,metadata_json FROM " + prefix + 'source_records LIMIT 1 RETURNING id', (value,)).fetchone()[0]
+            edition = pg.execute('SELECT id FROM ' + prefix + 'editions LIMIT 1').fetchone()[0]
+            submission = pg.execute('INSERT INTO ' + prefix + 'submissions(paper_id,edition_id,source_record_id,source_submission_id) VALUES(%s,%s,%s,%s) RETURNING id', (value, edition, record, 'new')).fetchone()[0]
+            decision = pg.execute('INSERT INTO ' + prefix + "decisions(submission_id,status,raw_value,evidence_url,observed_at) VALUES(%s,'unknown','fixture','https://example.org/new','2026-01-01T00:00:00Z') RETURNING id", (submission,)).fetchone()[0]
+            self.assertGreater(submission, 5000)
+            self.assertGreater(decision, 6000)
+        real = ReadConnection
+        with patch('paper_atlas.postgres.client.ReadConnection', lambda service: real(service, self.target)):
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                querying.main(['--pg-service', self.service, '--query', 'graph models', '--status', 'rejected', '--outcome', 'accepted'])
+            results = json.loads(output.getvalue())
+            self.assertEqual(len(results), 1)
+            self.assertEqual(results[0]['known_acceptances'][0]['venue'], 'COLM')
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                querying.main(['--pg-service', self.service, '--venue', 'conference on language modeling', '--tag', 'custom:topic=manual', '--paper', '1'])
+            detail = json.loads(output.getvalue())[0]
+            self.assertEqual(len(detail['submissions']), 2)
+            self.assertEqual({t['namespace'] for t in detail['tags']}, {'arxiv:category', 'custom:topic'})
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                querying.main(['--pg-service', self.service, '--stats'])
+            stats = json.loads(output.getvalue())
+            self.assertEqual(stats['papers'], 2)
+            self.assertEqual(stats['papers_with_known_acceptance'], 1)
+        reader = real(self.service, self.target)
+        try:
+            import psycopg
+            with self.assertRaises(psycopg.errors.ReadOnlySqlTransaction):
+                reader.execute('DELETE FROM papers')
+        finally:
+            reader.close()
+
+    def interrupt_after_first_table(self):
+        calls = 0
+        def interrupt(*args):
+            nonlocal calls
+            calls += 1
+            if calls == 3:
+                raise RuntimeError('controlled interruption')
+        with patch('paper_atlas.postgres.migration.free_space', side_effect=interrupt):
+            with self.assertRaisesRegex(RuntimeError, 'controlled interruption'):
+                self.migrate()
+        self.assertFalse(self.published())
+
+    def test_interrupted_copy_resumes_without_duplicate_rows(self):
+        self.interrupt_after_first_table()
+        self.assertEqual(self.migrate()['status'], 'complete')
+
+    def test_resume_rejects_a_changed_source_snapshot(self):
+        self.interrupt_after_first_table()
+        with closing(sqlite3.connect(self.db)) as source, source:
+            source.execute("UPDATE papers SET abstract='changed' WHERE id=1")
+        with self.assertRaisesRegex(ValueError, 'exact same'):
+            self.migrate()
+        self.assertFalse(self.published())
+
+    def test_full_hash_detects_tampering_in_a_completed_staging_table(self):
+        self.interrupt_after_first_table()
+        with connect(self.service) as pg:
+            stage = pg.execute('SELECT stage FROM ' + schema.quote(self.target + '_migration') + '.run WHERE id=1').fetchone()[0]
+            pg.execute('UPDATE ' + schema.quote(stage) + ".sources SET homepage='tampered'")
+        with self.assertRaisesRegex(ValueError, 'content verification'):
+            self.migrate()
+        self.assertFalse(self.published())
+
+    def test_foreign_key_violation_never_publishes_target(self):
+        with closing(sqlite3.connect(self.db)) as source, source:
+            source.execute('PRAGMA foreign_keys=OFF')
+            source.execute("INSERT INTO identifiers VALUES('orphan','bad',999999)")
+        import psycopg
+        with self.assertRaises(psycopg.errors.ForeignKeyViolation):
+            self.migrate()
+        self.assertFalse(self.published())
+
+    def test_published_target_cannot_be_overwritten(self):
+        self.migrate()
+        self.out.unlink()
+        with self.assertRaisesRegex(ValueError, 'never be overwritten'):
+            self.migrate()
+
+    def test_nul_and_literal_escape_survive_full_copy_keys_query_and_hash(self):
+        original = 'before\x00survivor; literal\x01\x01 and \x01\x02 — 研究'
+        keys = ['a\x00z', 'a\x01z', 'a\x01\x02z', 'a\x02z']
+        with closing(sqlite3.connect(self.db)) as source, source:
+            source.execute('UPDATE papers SET abstract=? WHERE id=1', (original,))
+            source.execute('UPDATE source_records SET abstract=? WHERE id=1', (original,))
+            source.executemany("INSERT INTO identifiers VALUES('control',?,1)", [(k,) for k in keys])
+            source.execute("INSERT INTO tags(namespace,value) VALUES('custom:control',?)", (original,))
+            tag = source.execute("SELECT id FROM tags WHERE namespace='custom:control'").fetchone()[0]
+            source.executemany('INSERT INTO paper_tags VALUES(1,?,NULL,?)', [(tag, k) for k in keys])
+        report = self.migrate()
+        self.assertEqual(report['text_codec'], text.CODEC)
+        for entry in report['tables'].values():
+            self.assertEqual(entry['source_sha256'], entry['target_sha256'])
+        self.assertNotEqual(report['tables']['source_records']['source_sha256'],
+                            report['tables']['source_records']['target_storage_sha256'])
+        with connect(self.service) as pg:
+            self.assertEqual(pg.execute("SELECT obj_description(oid,'pg_namespace') FROM pg_namespace WHERE nspname=%s", (self.target,)).fetchone()[0], 'paper-atlas:text-codec=' + text.CODEC)
+            raw = pg.execute('SELECT abstract FROM ' + schema.quote(self.target) + '.source_records WHERE id=1').fetchone()[0]
+            self.assertNotIn('\x00', raw)
+            self.assertEqual(text.decode(raw), original)
+        reader = ReadConnection(self.service, self.target)
+        try:
+            self.assertEqual(reader.execute('SELECT abstract FROM source_records WHERE id=?', (1,)).fetchone()[0], original)
+            self.assertEqual([r[0] for r in reader.execute("SELECT value FROM identifiers WHERE scheme='control' ORDER BY value")], sorted(keys))
+            self.assertEqual(reader.execute("SELECT value FROM identifiers WHERE scheme='control' AND value=?", (keys[0],)).fetchone()[0], keys[0])
+        finally:
+            reader.close()
+        real = ReadConnection
+        with patch('paper_atlas.postgres.client.ReadConnection', lambda service: real(service, self.target)):
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                querying.main(['--pg-service', self.service, '--paper', '1', '--tag', 'custom:control=' + original])
+            result = json.loads(output.getvalue())[0]
+            self.assertEqual(result['abstract'], original)
+            self.assertEqual(next(s['abstract'] for s in result['sources'] if s['external_id'] == '2401.00001'), original)
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                querying.main(['--pg-service', self.service, '--query', 'survivor'])
+            self.assertEqual([r['id'] for r in json.loads(output.getvalue())], [1])
+
+    def test_old_completed_staging_table_is_upgraded_atomically_on_resume(self):
+        original = 'https://example.org/literal\x01\x02'
+        with closing(sqlite3.connect(self.db)) as source, source:
+            source.execute('UPDATE sources SET homepage=? WHERE id=1', (original,))
+        self.interrupt_after_first_table()
+        with connect(self.service) as pg:
+            control = schema.quote(self.target + '_migration')
+            stage = pg.execute('SELECT stage FROM ' + control + '.run WHERE id=1').fetchone()[0]
+            # Reproduce the old implementation: raw SOH text and no codec metadata.
+            pg.execute('UPDATE ' + schema.quote(stage) + '.sources SET homepage=%s WHERE id=1', (original,))
+            pg.execute('ALTER TABLE ' + control + '.run DROP COLUMN text_codec')
+        report = self.migrate()
+        self.assertEqual(report['legacy_text_upgrade_rows']['sources'], 1)
+        self.assertEqual(report['tables']['sources']['source_sha256'], report['tables']['sources']['target_sha256'])
+        reader = ReadConnection(self.service, self.target)
+        try:
+            self.assertEqual(reader.execute('SELECT homepage FROM sources WHERE id=1').fetchone()[0], original)
+        finally:
+            reader.close()
+
+    def test_legacy_unique_keys_can_upgrade_after_constraints_were_built(self):
+        keys = ['x\x01', 'x\x01\x02']
+        with closing(sqlite3.connect(self.db)) as source, source:
+            source.executemany("INSERT INTO identifiers VALUES('control',?,1)", [(k,) for k in keys])
+        # Emulate an old run interrupted after constraints: no storage encoding.
+        with patch('paper_atlas.postgres.text.encode', side_effect=lambda value: value), \
+             patch('paper_atlas.postgres.migration.sequences', side_effect=RuntimeError('after constraints')):
+            with self.assertRaisesRegex(RuntimeError, 'after constraints'):
+                self.migrate()
+        with connect(self.service) as pg:
+            pg.execute('ALTER TABLE ' + schema.quote(self.target + '_migration') + '.run DROP COLUMN text_codec')
+        report = self.migrate()
+        self.assertEqual(report['status'], 'complete')
+        self.assertEqual(report['tables']['identifiers']['source_sha256'], report['tables']['identifiers']['target_sha256'])
+        reader = ReadConnection(self.service, self.target)
+        try:
+            self.assertEqual([r[0] for r in reader.execute("SELECT value FROM identifiers WHERE scheme='control' ORDER BY value")], keys)
+        finally:
+            reader.close()
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_postgres_ingestion.py

@@ -0,0 +1,298 @@
+from contextlib import closing, redirect_stdout
+import io
+import json
+import os
+from pathlib import Path
+import sqlite3
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+import uuid
+import zipfile
+
+from paper_atlas.db.backend import open_store, open_read_connection
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion import arxiv
+from paper_atlas.ingestion.backfill import backfill
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.postgres.client import connect, ReadConnection
+from paper_atlas.postgres.migration import migrate
+from paper_atlas.postgres.schema import quote
+
+
+class BackendSelectionTests(unittest.TestCase):
+    def test_ambiguous_targets_fail_before_opening_any_database(self):
+        for kwargs in ({}, {'db': 'example', 'pg_service': 'example'}, {'db': 'example', 'pg_schema': 'research'}):
+            with self.assertRaisesRegex(ValueError, 'Choose exactly one'):
+                open_store(**kwargs)
+
+    def test_readonly_audit_does_not_create_an_absent_sqlite_database(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / 'absent.sqlite'
+            with self.assertRaisesRegex(ValueError, 'Existing database'):
+                open_read_connection(path)
+            self.assertFalse(path.exists())
+
+
+@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the approved remote PostgreSQL test service')
+class PostgreSQLIngestionTests(unittest.TestCase):
+    def setUp(self):
+        self.service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
+        self.schema = 'pa_write_' + uuid.uuid4().hex[:12]
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        source = self.root / 'seed.sqlite'
+        store = Store(source)
+        paper, record = store.record('arxiv', '2401.00001', 'https://arxiv.org/abs/2401.00001',
+                                     'Existing reference paper', 'Existing abstract', [])
+        store.decision(paper, record, 'ICLR', 2024, 'main', 'seed', 'https://example.org/seed', 'rejected', 'Reject')
+        store.conn.execute("INSERT INTO paper_merges(former_paper_id,surviving_paper_id,reason) VALUES(900,?,'fixture')", (paper,))
+        store.conn.execute("INSERT INTO submission_merges VALUES(5000,1,'fixture',CURRENT_TIMESTAMP)")
+        store.conn.execute("INSERT INTO decision_merges VALUES(6000,1,'fixture',CURRENT_TIMESTAMP)")
+        store.conn.execute("UPDATE sqlite_sequence SET seq=1000 WHERE name='papers'")
+        store.conn.commit(); store.conn.close()
+        self.addCleanup(self.cleanup)
+        with redirect_stdout(io.StringIO()):
+            migrate(source, self.service, self.root / 'migration.json', schema=self.schema, min_free_gib=0)
+
+    def cleanup(self):
+        with connect(self.service) as pg:
+            names = pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (self.schema,)).fetchall()
+            for name, in names:
+                pg.execute('DROP SCHEMA ' + quote(name) + ' CASCADE')
+
+    def store(self):
+        return open_store(pg_service=self.service, pg_schema=self.schema)
+
+    def row(self, source='openreview_snapshot', external='forumNew1', status='rejected', year=2025, venue='ICLR'):
+        return {'source': source, 'external_id': external, 'url': 'https://example.org/' + external,
+                'venue': venue, 'year': year, 'track': 'main', 'status': status,
+                'title': 'A newly observed shared paper', 'abstract': 'Before\x00survivor and literal\x01\x02',
+                'authors': ['Jane Doe', 'John Smith'], 'identifiers': {'arxiv': '2501.99999'},
+                'observed_at': str(year) + '-01-20T00:00:00Z'}
+
+    def file(self, name, rows, count=None):
+        path = self.root / (name + '.jsonl')
+        path.write_text(''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in rows))
+        first = rows[0]
+        manifest = {key: first[key] for key in ('source', 'venue', 'year', 'track')}
+        manifest.update(parser_version=2, list_url='https://example.org/' + name,
+                        count=len(rows) if count is None else count, missing_abstract_count=0, status='complete')
+        path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
+        return path
+
+    def ingest(self, path):
+        with redirect_stdout(io.StringIO()):
+            ingest_file(None, path, pg_service=self.service, pg_schema=self.schema)
+
+    def test_native_writes_keep_nul_historical_ids_and_later_acceptance(self):
+        store = self.store()
+        try:
+            paper, record = store.record('public', 'new-forum', 'https://example.org/new', 'New research identity',
+                                         'Left\x00right\x01\x02', [], {'arxiv': '2501.99999'})
+            self.assertGreater(paper, 1000)
+            store.decision(paper, record, 'ICLR', 2025, 'main', 'new-forum', 'https://example.org/reject', 'rejected', 'Reject')
+            _, second = store.record('official', 'new-pub', 'https://example.org/pub', 'Changed title', None, [], {'arxiv': '2501.99999'})
+            store.decision(paper, second, 'COLM', 2026, 'main', 'new-pub', 'https://example.org/accept', 'accepted', 'Accept')
+            store.conn.commit()
+            self.assertEqual(store.conn.execute('SELECT abstract FROM papers WHERE id=?', (paper,)).fetchone()[0], 'Left\x00right\x01\x02')
+            ids = store.conn.execute('SELECT s.id,d.id FROM submissions s JOIN decisions d ON d.submission_id=s.id WHERE s.paper_id=?', (paper,)).fetchall()
+            self.assertEqual(len(ids), 2)
+            self.assertTrue(all(sid > 5000 and did > 6000 for sid, did in ids))
+        finally:
+            store.conn.close()
+        reader = ReadConnection(self.service, self.schema)
+        try:
+            from paper_atlas.querying import known_acceptances
+            self.assertEqual(known_acceptances(reader, paper)[0]['venue'], 'COLM')
+        finally:
+            reader.close()
+
+    def test_file_import_is_idempotent_and_refreshes_only_own_tags(self):
+        row = self.row(); row['tags'] = [{'namespace': 'openreview_snapshot:area', 'value': 'theory'}]
+        path = self.file('reject', [row]); self.ingest(path)
+        store = self.store()
+        try:
+            paper = store.conn.execute("SELECT paper_id FROM identifiers WHERE scheme='arxiv' AND value='2501.99999'").fetchone()[0]
+            store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','manual') ON CONFLICT DO NOTHING")
+            tag = store.conn.execute("SELECT id FROM tags WHERE namespace='custom:topic'").fetchone()[0]
+            store.conn.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)', (paper, tag, 'manual'))
+            store.conn.commit()
+        finally:
+            store.conn.close()
+        row['tags'][0]['value'] = 'learning'; self.ingest(self.file('reject', [row]))
+        self.ingest(self.file('accept', [self.row('official', 'published', 'accepted', 2026, 'COLM')]))
+        store = self.store()
+        try:
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 2)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions WHERE paper_id=?', (paper,)).fetchone()[0], 2)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id WHERE s.paper_id=?', (paper,)).fetchone()[0], 2)
+            tags = set(store.conn.execute('SELECT t.namespace,t.value FROM paper_tags p JOIN tags t ON t.id=p.tag_id WHERE p.paper_id=?', (paper,)))
+            self.assertEqual(tags, {('custom:topic', 'manual'), ('openreview_snapshot:area', 'learning')})
+        finally:
+            store.conn.close()
+
+    def test_bad_manifest_rolls_back_whole_file_and_keeps_failed_run(self):
+        with self.assertRaisesRegex(ValueError, 'Manifest count'):
+            self.ingest(self.file('bad', [self.row()], count=2))
+        store = self.store()
+        try:
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0], 1)
+            self.assertEqual(store.conn.execute('SELECT status,records_seen,records_written FROM ingest_runs').fetchone(), ('failed', 1, 0))
+        finally:
+            store.conn.close()
+
+    def test_changed_file_hash_rolls_back_data_before_commit(self):
+        path = self.file('changed', [self.row()])
+        manifest = json.loads(path.with_suffix('.manifest.json').read_text())
+        manifest['file_sha256'] = '0' * 64
+        path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
+        with self.assertRaisesRegex(ValueError, 'hash differs'):
+            self.ingest(path)
+        store = self.store()
+        try:
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0], 1)
+            self.assertEqual(store.conn.execute('SELECT status,records_written FROM ingest_runs').fetchone(), ('failed', 0))
+        finally:
+            store.conn.close()
+
+    def test_writer_lock_survives_commit_and_rollback_but_readers_continue(self):
+        first = self.store()
+        try:
+            first.conn.commit()
+            with self.assertRaisesRegex(RuntimeError, 'Another writer'):
+                self.store()
+            first.conn.rollback()
+            with self.assertRaisesRegex(RuntimeError, 'Another writer'):
+                self.store()
+            reader = ReadConnection(self.service, self.schema)
+            try:
+                self.assertEqual(reader.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+            finally:
+                reader.close()
+        finally:
+            first.conn.close()
+        reopened = self.store(); reopened.conn.close()
+
+    def test_unpublished_schema_is_rejected_without_bootstrapping_tables(self):
+        staging = self.schema + '_unfinished'
+        with connect(self.service) as pg:
+            pg.execute('CREATE SCHEMA ' + quote(staging))
+        with self.assertRaisesRegex(RuntimeError, 'published dataset'):
+            open_store(pg_service=self.service, pg_schema=staging)
+        with connect(self.service) as pg:
+            self.assertEqual(pg.execute('SELECT count(*) FROM information_schema.tables WHERE table_schema=%s', (staging,)).fetchone()[0], 0)
+
+    def test_negative_preflight_uses_current_pg_attempt_and_blocks_stale_import(self):
+        import hashlib
+        from paper_atlas.collectors.public_openreview import SOURCE, REVISION
+        from paper_atlas.audits.public_openreview import audit
+        row = self.row(SOURCE, 'negativeForum')
+        row['identifiers']['openreview'] = 'negativeForum'
+        row['metadata'] = {'evidence_kind': 'curated_public_status_assertion',
+                           'source_revision': REVISION, 'original_decision_signature_verified': False}
+        path = self.file('negative', [row])
+        manifest = json.loads(path.with_suffix('.manifest.json').read_text())
+        manifest.update(file=str(path), file_sha256=hashlib.sha256(path.read_bytes()).hexdigest())
+        path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
+        inventory = self.root / 'negative-inventory.json'
+        inventory.write_text(json.dumps({'source': SOURCE, 'revision': REVISION,
+                                        'normalized_records': 1, 'files': [manifest]}))
+        # An acceptance in another conference does not block this rejection.
+        self.ingest(self.file('other-conference', [self.row('official', 'acceptedElsewhere', 'accepted', 2026, 'COLM')]))
+        def preflight(name):
+            with redirect_stdout(io.StringIO()):
+                return audit(None, inventory, self.root / (name + '.json'), pg_service=self.service, pg_schema=self.schema)
+        first = preflight('before')
+        self.assertEqual(first['status'], 'ready_for_serial_import')
+        self.assertEqual(first['engine'], 'postgresql')
+        self.assertEqual(first['schema'], self.schema)
+        self.ingest(path)
+        self.assertEqual(preflight('existing')['existing_same_edition_attempts'], 1)
+        store = self.store()
+        try:
+            paper, record = store.conn.execute('''SELECT r.paper_id,r.id FROM source_records r
+                JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?''', (SOURCE, row['external_id'])).fetchone()
+            store.decision(paper, record, 'ICLR', 2025, 'main', row['external_id'],
+                           'https://example.org/formal', 'accepted', 'Accept', observed_at='2027-01-01T00:00:00Z')
+            # Equal observation time, but an explicitly earlier decision date.
+            # NULL dates must fall back to observed_at, not the empty string.
+            store.decision(paper, record, 'ICLR', 2025, 'main', row['external_id'],
+                           'https://example.org/older', 'rejected', 'Reject', observed_at='2027-01-01T00:00:00Z',
+                           decision_date='2026-12-31T00:00:00Z')
+            store.conn.commit()
+        finally:
+            store.conn.close()
+        blocked = preflight('blocked')
+        self.assertEqual(blocked['status'], 'blocked')
+        self.assertEqual(blocked['conflicts'][0]['existing_status'], 'accepted')
+        self.assertEqual(blocked['files'], [])
+        # The earlier ready report cannot authorize overwriting current status.
+        with self.assertRaisesRegex(ValueError, 'conflicts with current'):
+            self.ingest(path)
+        reader = open_read_connection(pg_service=self.service, pg_schema=self.schema)
+        try:
+            self.assertEqual(reader.execute('SHOW transaction_read_only').fetchone()[0], 'on')
+            from paper_atlas.db.submissions import latest_status
+            sid = reader.execute("SELECT submission_id FROM submission_identifiers WHERE scheme='openreview' AND value=?", ('negativeForum',)).fetchone()[0]
+            self.assertEqual(latest_status(reader, sid), 'accepted')
+        finally:
+            reader.close()
+
+    def test_backfill_merges_evidence_without_reimporting_or_losing_history(self):
+        files = []
+        for source, external, title in [('official', 'legacy', 'First bibliographic name'), ('openreview_snapshot', 'forum', 'Renamed submission')]:
+            row = self.row(source, external, 'accepted')
+            row.update(title=title, authors=[], identifiers={}, evidence_url='https://example.org/same-decision')
+            path = self.file(source, [row]); self.ingest(path)
+            row['identifiers'] = {'openreview': 'sharedForum'}
+            self.file(source, [row]); files.append(path)
+        with redirect_stdout(io.StringIO()):
+            result = backfill(None, files, pg_service=self.service, pg_schema=self.schema)
+        self.assertEqual((result['submissions_before'], result['submissions_after']), (3, 2))
+        store = self.store()
+        try:
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 2)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM decision_sources').fetchone()[0], 3)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM decision_merges').fetchone()[0], 2)
+        finally:
+            store.conn.close()
+
+    def test_arxiv_chunk_resume_preserves_committed_ids_and_category_evidence(self):
+        archive = self.root / 'arxiv.zip'
+        rows = [{'id': f'2501.{i:05d}', 'title': f'Checkpoint paper {i}', 'abstract': 'Metadata abstract',
+                 'authors': [], 'categories': 'cs.AI'} for i in range(1, 502)]
+        with zipfile.ZipFile(archive, 'w') as zipped:
+            zipped.writestr('arxiv-metadata-oai-snapshot.json', ''.join(json.dumps(r) + '\n' for r in rows))
+        calls = 0
+        original = arxiv.refresh_categories
+        def interrupt(*args):
+            nonlocal calls
+            calls += 1
+            if calls == 501:
+                raise RuntimeError('controlled interruption')
+            return original(*args)
+        with patch('paper_atlas.ingestion.arxiv.shutil.disk_usage', return_value=SimpleNamespace(free=8 * 1024**3)), \
+             patch('paper_atlas.ingestion.arxiv.refresh_categories', side_effect=interrupt), redirect_stdout(io.StringIO()):
+            with self.assertRaisesRegex(RuntimeError, 'controlled interruption'):
+                arxiv.ingest(None, archive, pg_service=self.service, pg_schema=self.schema)
+        store = self.store()
+        try:
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 501)
+            preserved = store.conn.execute("SELECT paper_id FROM identifiers WHERE scheme='arxiv' AND value='2501.00001'").fetchone()[0]
+        finally:
+            store.conn.close()
+        with redirect_stdout(io.StringIO()):
+            arxiv.ingest(None, archive, pg_service=self.service, pg_schema=self.schema)
+        store = self.store()
+        try:
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 502)
+            self.assertEqual(store.conn.execute("SELECT paper_id FROM identifiers WHERE scheme='arxiv' AND value='2501.00001'").fetchone()[0], preserved)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_tags').fetchone()[0], 501)
+            self.assertEqual(store.conn.execute('SELECT status,records_written FROM ingest_runs').fetchone(), ('complete', 501))
+        finally:
+            store.conn.close()

tests/test_proceedings.py

@@ -1,11 +1,10 @@
 import unittest
 import json
-import runpy
 import tempfile
 from pathlib import Path
 from unittest.mock import patch
 
-from collect_proceedings import paper_links,parse_paper
+from paper_atlas.collectors.proceedings import paper_links,parse_paper,main
 
 
 class ProceedingsTests(unittest.TestCase):
@@ -50,9 +49,9 @@ class ProceedingsTests(unittest.TestCase):
         with tempfile.TemporaryDirectory() as tmp:
             status=Path(tmp)/'colm-index-status.json'
             status.write_text(json.dumps({other:{'status':'collected','files':[]}}))
-            argv=['collect_proceedings.py','--source','colm','--cache',tmp,'--out',tmp,'--year','2024']
-            with patch('sys.argv',argv),patch('collect_pmlr.fetch',side_effect=lambda u,*a,**kw:documents[u].encode()):
-                runpy.run_path(str(Path(__file__).with_name('collect_proceedings.py')),run_name='__main__')
+            argv=['paper-atlas','--source','colm','--cache',tmp,'--out',tmp,'--year','2024']
+            with patch('sys.argv',argv),patch('paper_atlas.collectors.proceedings.fetch',side_effect=lambda u,*a,**kw:documents[u].encode()):
+                main()
             result=json.loads(status.read_text())
             self.assertEqual(result[other]['status'],'collected')
             self.assertEqual(result[url]['files'][0]['count'],1)

tests/test_public_openreview.py

@@ -0,0 +1,186 @@
+import hashlib
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors.public_openreview import (REVISION, SOURCE, collect,
+                                                     negative_status, normalize)
+from paper_atlas.db.store import Store
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.audits.public_openreview import audit
+
+
+class PublicOpenReviewTests(unittest.TestCase):
+    def row(self, **overrides):
+        return {'id': 'RealForum01', 'title': 'A public paper', 'abstract': 'An abstract',
+                'author': 'Anonymous;Jane Doe;private@example.org', 'track': 'main',
+                'status': 'Reject', 'keywords': 'planning;reasoning',
+                'profiles': {'private': 'secret'}, 'reviews': ['private review'],
+                'gender': 'irrelevant', **overrides}
+
+    def provenance(self):
+        return {'source_file': 'https://example.org/pinned.jsonl', 'source_line': 7,
+                'source_revision': REVISION, 'source_blob_sha1': 'a' * 40}
+
+    def fetcher(self, files):
+        tree = []
+        payloads = {}
+        for name, rows in files.items():
+            data = ''.join(json.dumps(row) + '\n' for row in rows).encode()
+            sha = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
+            tree.append({'path': name, 'type': 'blob', 'size': len(data), 'sha': sha})
+            payloads[name] = data
+
+        def fetch(url, limit):
+            if '?recursive=1' in url:
+                return json.dumps({'sha': REVISION, 'truncated': False, 'tree': tree}).encode()
+            return payloads[url.split('/' + REVISION + '/')[1]]
+        return fetch
+
+    def collect(self, out, files, **kwargs):
+        with patch('paper_atlas.collectors.public_openreview.shutil.disk_usage', return_value=type('Space', (), {'free': 20 * 1024**3})()):
+            return collect(out, fetcher=self.fetcher(files), **kwargs)
+
+    def test_only_explicit_decisions_and_verified_scope_become_events(self):
+        for value in ('Pending', 'Active', 'Submitted to NeurIPS 2025', 'TBD', '',
+                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject'):
+            self.assertIsNone(negative_status(value), value)
+        self.assertEqual(negative_status('Reject'), 'rejected')
+        self.assertEqual(negative_status('Desk rejected'), 'desk_rejected')
+        self.assertEqual(negative_status('NeurIPS 2023 Conference Withdrawn Submission'), 'withdrawn')
+        for raw, expected in [('main', 'main'), ('Position', 'position'),
+                              ('Datasets & Benchmarks', 'datasets_benchmarks')]:
+            row, reason = normalize(self.row(track=raw), 'nips', 2025, self.provenance())
+            self.assertIsNone(reason)
+            self.assertEqual(row['track'], expected)
+        for raw in ('workshop', 'workshop_mlsys', ''):
+            self.assertIsNone(normalize(self.row(track=raw), 'icml', 2025, self.provenance())[0])
+        with self.assertRaisesRegex(ValueError, 'conference year'):
+            normalize(self.row(status='NeurIPS 2023 Conference Withdrawn Submission'), 'icml', 2025, self.provenance())
+
+    def test_metadata_whitelist_retains_curated_provenance_without_profiles_or_reviews(self):
+        row, _ = normalize(self.row(), 'icml', 2025, self.provenance())
+        self.assertEqual(row['authors'], ['Jane Doe'])
+        self.assertEqual(row['identifiers'], {'openreview': 'RealForum01'})
+        self.assertEqual(row['metadata']['evidence_kind'], 'curated_public_status_assertion')
+        self.assertFalse(row['metadata']['original_decision_signature_verified'])
+        self.assertEqual(row['evidence_url'], 'https://example.org/pinned.jsonl#L7')
+        for private in ('secret', 'private review', 'gender', 'private@example.org'):
+            self.assertNotIn(private, json.dumps(row))
+
+    def test_pinned_collection_separates_tracks_and_holds_retraction(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            files = {'venues/icml/icml2025.jsonl': [self.row(), self.row(id='Position01', track='Position'),
+                     self.row(id='Retracted1', status='ICML 2025 Retracted by Authors'),
+                     self.row(id='Pending001', status='Submitted to ICML 2025')]}
+            result = self.collect(tmp, files)
+            self.assertEqual(result['input_rows'], 4)
+            self.assertEqual(result['normalized_records'], 2)
+            self.assertEqual({f['track'] for f in result['files']}, {'main', 'position'})
+            held = [json.loads(line) for line in Path(result['held_file']).read_text().splitlines()]
+            self.assertEqual([(r['id'], r['reason']) for r in held], [('Retracted1', 'retracted')])
+            self.assertTrue(all(f['status'] == 'partial' for f in result['files']))
+            with self.assertRaisesRegex(ValueError, 'Completed inventory'):
+                self.collect(tmp, files)
+
+    def test_hash_failure_or_duplicate_forum_publishes_no_ingestible_files(self):
+        files = {'venues/icml/icml2025.jsonl': [self.row()]}
+        fetch = self.fetcher(files)
+        def corrupted(url, limit):
+            result = fetch(url, limit)
+            return result if '?recursive' in url else result.replace(b'Reject', b'Accept')
+        for fetcher in (corrupted, self.fetcher({'venues/icml/icml2025.jsonl': [self.row(), self.row()]})):
+            with tempfile.TemporaryDirectory() as tmp, patch('paper_atlas.collectors.public_openreview.shutil.disk_usage', return_value=type('Space', (), {'free': 20 * 1024**3})()):
+                with self.assertRaises(ValueError):
+                    collect(tmp, fetcher=fetcher)
+                self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
+                self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
+
+    def test_mirror_reimport_preserves_one_attempt_and_later_acceptance_elsewhere(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            result = self.collect(root / 'source', {'venues/icml/icml2025.jsonl': [self.row(author='Jane Doe')]})
+            db = root / 'db.sqlite'
+            path = result['files'][0]['file']
+            ingest_file(db, path)
+            ingest_file(db, path)
+            store = Store(db)
+            try:
+                paper = store.conn.execute('SELECT paper_id FROM source_records').fetchone()[0]
+                store.conn.execute('INSERT INTO identifiers VALUES(?,?,?)', ('arxiv', '2501.00001', paper))
+                same, record = store.record('official_program', 'neurips-later', 'https://example.org/accepted',
+                                            'A revised public paper', 'An abstract', ['Jane Doe'],
+                                            {'openreview': 'LaterForum1', 'arxiv': '2501.00001'})
+                self.assertEqual(same, paper)
+                store.decision(paper, record, 'NeurIPS', 2026, 'main', 'neurips-later',
+                               'https://example.org/accepted', 'accepted', 'Accept',
+                               identities=[('openreview', 'LaterForum1')])
+                store.conn.commit()
+                self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0], 2)
+                self.assertEqual(store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0], 2)
+                self.assertEqual(set(store.conn.execute('SELECT status FROM decisions')), {('accepted',), ('rejected',)})
+                self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
+                self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(), [])
+            finally:
+                store.conn.close()
+
+    def test_preimport_audit_blocks_same_attempt_conflict_but_allows_later_other_venue_acceptance(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            self.collect(root / 'source', {'venues/icml/icml2025.jsonl': [self.row(author='Jane Doe')]})
+            db = root / 'db.sqlite'
+            store = Store(db)
+            paper, record = store.record('program', 'later', 'https://example.org/later',
+                                         'A public paper', 'An abstract', ['Jane Doe'], {'openreview': 'LaterForum1'})
+            store.decision(paper, record, 'NeurIPS', 2026, 'main', 'later',
+                           'https://example.org/later', 'accepted', 'Accept', identities=[('openreview', 'LaterForum1')])
+            store.conn.commit()
+            inventory = root / 'source/public-openreview-inventory.json'
+            first = audit(db, inventory, root / 'audit.json')
+            self.assertEqual(first['status'], 'ready_for_serial_import')
+            self.assertEqual(len(first['files']), 1)
+            store.decision(paper, record, 'ICML', 2025, 'main', 'same-attempt',
+                           'https://example.org/same', 'accepted', 'Accept', identities=[('openreview', 'RealForum01')])
+            store.conn.commit()
+            try:
+                second = audit(db, inventory, root / 'audit.json')
+                self.assertEqual(second['status'], 'blocked')
+                self.assertEqual(second['files'], [])
+                self.assertEqual(second['conflicts'][0]['existing_status'], 'accepted')
+            finally:
+                store.conn.close()
+
+    def test_unicode_separators_in_abstract_survive_audit_and_import(self):
+        for separator in ('\u0085', '\u2028', '\u2029'):
+            with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                abstract = 'First paragraph' + separator + 'Second paragraph'
+                result = self.collect(root / 'source', {
+                    'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]})
+                path = Path(result['files'][0]['file'])
+                data = path.read_bytes()
+                self.assertIn(separator.encode(), data)
+                self.assertEqual(data.count(b'\n'), 1)
+                db = root / 'db.sqlite'
+                store = Store(db)
+                store.conn.close()
+                inventory = root / 'source/public-openreview-inventory.json'
+                before = audit(db, inventory, root / 'before.json')
+                self.assertEqual(before['status'], 'ready_for_serial_import')
+                self.assertEqual(before['records_checked'], 1)
+                ingest_file(db, path)
+                after = audit(db, inventory, root / 'after.json')
+                self.assertEqual(after['status'], 'ready_for_serial_import')
+                self.assertEqual(after['existing_same_edition_attempts'], 1)
+                store = Store(db)
+                try:
+                    self.assertEqual(store.conn.execute(
+                        'SELECT abstract FROM source_records').fetchone()[0], abstract)
+                finally:
+                    store.conn.close()
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_query.py

@@ -0,0 +1,86 @@
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+from paper_atlas.db.store import Store
+
+
+class QueryTests(unittest.TestCase):
+    def run_query(self,db,*args):
+        return json.loads(subprocess.run([sys.executable,'-B','-m','paper_atlas','query','--db',str(db),*args],check=True,capture_output=True,text=True).stdout)
+
+    def test_rejection_then_acceptance_at_another_venue_preserves_both_attempts(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            db=Path(tmp)/'papers.sqlite';store=Store(db);self.addCleanup(store.conn.close)
+            paper,first=store.record('openreview_snapshot','forum1','https://example.org/1','Original title','Abstract',['Jane Doe'],{'arxiv':'2401.00001','openreview':'forum1'})
+            store.decision(paper,first,'ICLR',2024,'main','forum1','https://example.org/reject','rejected','Reject',identities=[('openreview','forum1')],observed_at='2024-02-01T00:00:00Z')
+            same,second=store.record('colm','forum2','https://example.org/2','Revised title','Revised abstract',['Jane Doe'],{'arxiv':'2401.00001','openreview':'forum2'})
+            store.decision(same,second,'COLM',2025,'main','forum2','https://example.org/accept','accepted','Accept',identities=[('openreview','forum2')],observed_at='2025-08-01T00:00:00Z')
+            store.conn.commit();self.assertEqual(same,paper)
+            rows=self.run_query(db,'--venue','ICLR','--status','rejected','--outcome','accepted')
+            self.assertEqual([row['id'] for row in rows],[paper])
+            self.assertEqual(rows[0]['publication_outcome'],'accepted')
+            self.assertEqual([a['venue'] for a in rows[0]['known_acceptances']],['COLM'])
+            detail=self.run_query(db,'--paper',str(paper))[0]
+            self.assertEqual([(s['acronym'],s['latest_status']) for s in detail['submissions']],[('ICLR','rejected'),('COLM','accepted')])
+            self.assertEqual(self.run_query(db,'--venue','ICLR','--status','rejected','--outcome','no_known_acceptance'),[])
+            self.assertEqual(self.run_query(db,'--stats')['papers_with_known_acceptance'],1)
+
+    def test_later_rejection_elsewhere_does_not_erase_acceptance_and_unknown_is_not_never(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            db=Path(tmp)/'papers.sqlite';store=Store(db);self.addCleanup(store.conn.close)
+            paper,one=store.record('source','one','https://example.org/1','One paper','Abstract',[])
+            store.decision(paper,one,'COLM',2024,'main','one','https://example.org/accept','accepted','Accept',observed_at='2024-01-01T00:00:00Z')
+            store.decision(paper,one,'ICLR',2025,'main','two','https://example.org/reject','rejected','Reject',observed_at='2025-01-01T00:00:00Z')
+            unknown,_=store.record('arxiv','other','https://example.org/other','No observed submissions','Abstract',[])
+            store.conn.commit()
+            self.assertEqual([r['id'] for r in self.run_query(db,'--outcome','accepted')],[paper])
+            self.assertEqual([r['id'] for r in self.run_query(db,'--outcome','no_known_acceptance')],[unknown])
+            store.decision(paper,one,'COLM',2024,'main','one','https://example.org/withdraw','withdrawn','Withdrawn',observed_at='2026-01-01T00:00:00Z')
+            store.conn.commit();self.assertEqual(self.run_query(db,'--outcome','accepted'),[])
+            history=self.run_query(db,'--paper',str(paper))[0]
+            self.assertEqual([d['status'] for d in history['submissions'][0]['decisions']],['accepted','withdrawn'])
+
+    def test_joint_conference_alias_does_not_duplicate_submission(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            db=Path(tmp)/'papers.sqlite';store=Store(db);self.addCleanup(store.conn.close)
+            paper,record=store.record('acl_anthology','paper1','https://example.org/1','Joint event paper','Abstract',[])
+            store.decision(paper,record,'COLING+LREC',2024,'main','paper1','https://example.org/1','accepted','official XML',['COLING','LREC'])
+            store.conn.commit()
+            for venue in ('COLING','LREC'):
+                command=[sys.executable,'-B','-m','paper_atlas','query','--db',str(db),'--venue',venue,'--track','main']
+                rows=json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout)
+                self.assertEqual([r['id'] for r in rows],[paper])
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
+
+    def test_venue_filter_is_case_insensitive_and_uses_latest_decision(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            db=Path(tmp)/'papers.sqlite'
+            store=Store(db)
+            self.addCleanup(store.conn.close)
+            paper,record=store.record('peerread','paper1','https://example.org/paper1','A real submission','Abstract',['Jane Doe'])
+            store.decision(paper,record,'ICLR',2017,'main','paper1','https://example.org/paper1','rejected','explicit rejection')
+            store.record('arxiv','2601.00001','https://example.org/other','Another paper','Abstract',['John Smith'])
+            store.conn.commit()
+            command=[sys.executable,'-B','-m','paper_atlas','query','--db',str(db),'--venue','iclr','--year','2017','--status','rejected']
+            result=json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout)
+            self.assertEqual([r['id'] for r in result],[paper])
+            details_command=[sys.executable,'-B','-m','paper_atlas','query','--db',str(db),'--paper',str(paper)]
+            details=json.loads(subprocess.run(details_command,check=True,capture_output=True,text=True).stdout)[0]
+            self.assertEqual(details['authors'],['Jane Doe'])
+            self.assertEqual(details['submissions'][0]['decisions'][0]['status'],'rejected')
+            self.assertEqual(details['sources'][0]['external_id'],'paper1')
+            stats_command=[sys.executable,'-B','-m','paper_atlas','query','--db',str(db),'--stats']
+            stats=json.loads(subprocess.run(stats_command,check=True,capture_output=True,text=True).stdout)
+            self.assertEqual(stats['papers'],2)
+            self.assertEqual({r['name']:r['imported_records'] for r in stats['records_by_source']},{'arxiv':1,'peerread':1})
+            store.decision(paper,record,'ICLR',2017,'main','paper1','https://example.org/paper1','accepted','changed decision')
+            store.conn.commit()
+            self.assertEqual(json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout),[])
+
+
+if __name__=='__main__':
+    unittest.main()

tests/test_submission_evidence.py

@@ -1,3 +1,4 @@
+from importlib.resources import files
 import json
 from pathlib import Path
 import sqlite3
@@ -6,10 +7,10 @@ import sys
 import tempfile
 import unittest
 
-from import_records import ingest_file
-from index_core import Store
-from submission_evidence import identities_for_row
-from backfill_submission_evidence import backfill,queue_files
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.db.store import Store
+from paper_atlas.db.submissions import identities_for_row
+from paper_atlas.ingestion.backfill import backfill,queue_files
 
 
 class SubmissionTests(unittest.TestCase):
@@ -25,7 +26,7 @@ class SubmissionTests(unittest.TestCase):
 
     def query(self,*args):
         self.store.conn.commit()
-        command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(self.db),*args]
+        command=[sys.executable,'-B','-m','paper_atlas','query','--db',str(self.db),*args]
         result=subprocess.run(command,capture_output=True,text=True)
         self.assertEqual(result.returncode,0,result.stderr)
         return json.loads(result.stdout)
@@ -125,7 +126,7 @@ class SubmissionTests(unittest.TestCase):
             identities_for_row({'source':'snapshot','venue':'ICLR','submission_identifiers':[{'scheme':'openreview','value':'2026-Oral--42-placeholder'}]})
 
     def test_legacy_database_migration_preserves_source_and_decision_links(self):
-        legacy=Path(self.tmp.name)/'legacy.sqlite';sql=Path(__file__).with_name('schema.sql').read_text()
+        legacy=Path(self.tmp.name)/'legacy.sqlite';sql=files('paper_atlas.db').joinpath('schema.sql').read_text()
         sql=sql[:sql.index('CREATE TABLE IF NOT EXISTS submission_sources')]+sql[sql.index('CREATE TABLE IF NOT EXISTS tags'):]
         c=sqlite3.connect(legacy);c.executescript(sql)
         c.execute("INSERT INTO sources VALUES(1,'legacy','https://example.org')")

tests/test_vision.py

@@ -0,0 +1,146 @@
+import tempfile
+import io
+import json
+from pathlib import Path
+import unittest
+from unittest.mock import patch
+from urllib.error import HTTPError
+
+from paper_atlas.collectors.vision import collect_index, decode_html, discover, index_entries, parse_paper,main
+
+
+class VisionTests(unittest.TestCase):
+    def cvf_fixture(self,year=2018):
+        root=f'https://openaccess.thecvf.com/CVPR{year}.py'
+        papers=[f'https://openaccess.thecvf.com/content_cvpr_{year}/html/Paper{i}_CVPR_{year}_paper.html' for i in (1,2,3)]
+        def entry(url,title='Paper'):
+            return f'<dt><a href="{url}">{title}</a></dt><dd>Jane Doe, John Smith</dd>'
+        return root,papers,entry
+
+    def test_all_explicit_days_are_combined_and_overlapping_papers_are_not_duplicated(self):
+        root,papers,entry=self.cvf_fixture();day1=root+'?day=2018-06-19';day2=root+'?day=2018-06-20'
+        pages={root:f'<a href="{day1}">Day1</a><a href="{day2}">Day2</a><a href="https://evil.example/CVPR2018.py?day=all">Other</a><a href="/CVPR2019.py?day=all">Wrong year</a>',day1:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>',day2:'<dl>'+entry(papers[1])+entry(papers[2])+'</dl>'}
+        pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers})
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
+            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+            rows=[json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
+        self.assertEqual(manifest['count'],3);self.assertEqual(manifest['raw_index_entry_count'],4)
+        self.assertEqual(manifest['index_urls'],[day1,day2]);self.assertEqual(manifest['status'],'collected')
+        self.assertEqual(rows[1]['metadata']['index_urls'],[day1,day2])
+        self.assertEqual({call.args[0] for call in fetch.call_args_list},set(pages))
+
+    def test_conflicting_copies_across_days_fail_before_publishing(self):
+        root,papers,entry=self.cvf_fixture();days=[root+'?day=2018-06-19',root+'?day=2018-06-20']
+        pages={root:''.join(f'<a href="{day}">Day</a>' for day in days),days[0]:'<dl>'+entry(papers[0],'First title')+'</dl>',days[1]:'<dl>'+entry(papers[0],'Different title')+'</dl>'}
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()):
+            with self.assertRaisesRegex(ValueError,'Conflicting metadata'):
+                collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+            self.assertFalse(list(Path(tmp).glob('*.jsonl')))
+
+    def test_missing_abstract_page_keeps_official_index_metadata_and_marks_partial(self):
+        root,papers,entry=self.cvf_fixture();error=HTTPError(papers[0],404,'Not Found',{},io.BytesIO())
+        def fetch(url,*args,**kwargs):
+            if url==root:return ('<dl>'+entry(papers[0],'Known publication')+'</dl>').encode()
+            raise error
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
+            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+            row=json.loads(Path(manifest['file']).read_text())
+        self.assertEqual(row['title'],'Known publication');self.assertIsNone(row['abstract'])
+        self.assertEqual(row['authors'],['Jane Doe','John Smith']);self.assertEqual(row['evidence_url'],root)
+        self.assertEqual(manifest['status'],'partial');self.assertEqual(manifest['missing_abstract_pages'],[papers[0]])
+        self.assertEqual(row['metadata']['abstract_fetch_error']['status'],404)
+
+    def test_forbidden_paper_is_not_replaced_with_a_fallback(self):
+        root,papers,entry=self.cvf_fixture()
+        def fetch(url,*args,**kwargs):
+            if url==root:return ('<dl>'+entry(papers[0])+'</dl>').encode()
+            raise HTTPError(url,403,'Forbidden',{},io.BytesIO())
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
+            with self.assertRaises(HTTPError):collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+            self.assertFalse(list(Path(tmp).glob('*.jsonl')))
+
+    def test_broken_all_papers_link_can_use_all_advertised_days(self):
+        root,papers,entry=self.cvf_fixture();day=root+'?day=2018-06-19';all_url=root+'?day=all'
+        pages={root:f'<a href="{day}">Day</a><a href="{all_url}">All</a>',day:'<dl>'+entry(papers[0])+'</dl>',papers[0]:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>'}
+        def fetch(url,*a,**kw):
+            if url==all_url:raise HTTPError(url,404,'Not Found',{},io.BytesIO())
+            return pages[url].encode()
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
+            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+        self.assertEqual(manifest['index_urls'],[day]);self.assertEqual(manifest['index_errors'],[{'url':all_url,'status':404}])
+
+    def test_selective_retry_uses_separate_output_and_leaves_original_catalogue_intact(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            old=Path(tmp)/'old.json';payload={'CVPR:2018:main':{'status':'blocked'},'CVPR:2019:main':{'status':'collected'}};old.write_text(json.dumps(payload))
+            indexes=[{'venue':'CVPR','year':year,'track':'main','index_url':f'https://openaccess.thecvf.com/CVPR{year}'} for year in (2018,2019)]
+            output=Path(tmp)/'repaired'
+            with patch('paper_atlas.collectors.vision.discover',return_value=indexes),patch('paper_atlas.collectors.vision.collect_index',return_value={'status':'collected','count':1}) as collect:
+                main(['--cache',tmp,'--out',str(output),'--retry-catalogue',str(old)],source='cvf')
+            self.assertEqual(collect.call_count,1);self.assertEqual(collect.call_args.args[1]['year'],2018)
+            self.assertEqual(json.loads(old.read_text()),payload)
+            self.assertEqual(set(json.loads((output/'cvf-index-status.json').read_text())),{'CVPR:2018:main'})
+
+    def entry(self):
+        return {'url':'https://www.ecva.net/papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php','index_title':'Paper','index_authors':'Jane Doe*, John Smith','doi':'10.1007/example_1'}
+
+    def test_ecva_doi_belongs_to_its_own_title_entry(self):
+        html='<dl><dt class="ptitle"><a href="papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php">First paper</a></dt><dd>Jane Doe*, John Smith</dd><dd><a href="https://link.springer.com/chapter/10.1007/example_1">DOI</a></dd><dt class="ptitle"><a href="papers/eccv_2024/papers_ECCV/html/2_ECCV_2024_paper.php">Second paper</a></dt><dd>Other Author</dd><dd><a href="https://doi.org/10.1007/example_2">DOI</a></dd></dl>'
+        entries,count=index_entries(html,'https://www.ecva.net/papers.php')
+        self.assertEqual(count,2)
+        self.assertEqual([e['doi'] for e in entries],['10.1007/example_1','10.1007/example_2'])
+        html='<div id="papertitle">Paper</div><div id="abstract">Abstract citing <a href="https://arxiv.org/abs/1706.03762">another paper</a> and <a href="https://doi.org/10.1007/wrong">a reference</a>.</div>'
+        row=parse_paper(html,entries[0],'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php')
+        self.assertEqual(row['identifiers'],{'doi':'10.1007/example_1'})
+        self.assertEqual(row['authors'],['Jane Doe','John Smith'])
+
+    def test_cvf_meta_authors_and_missing_abstract_never_capture_footer(self):
+        html='<meta name="citation_title" content="A vision paper"><meta name="citation_author" content="Doe, Jane"><div id="authors">Do not include the proceedings citation</div><p>Copyright footer</p>'
+        entry=dict(self.entry(),doi=None)
+        row=parse_paper(html,entry,'cvf','CVPR',2013,'main','https://openaccess.thecvf.com/CVPR2013')
+        self.assertEqual(row['authors'],['Jane Doe'])
+        self.assertIsNone(row['abstract'])
+        self.assertEqual(row['title'],'A vision paper')
+
+    def test_conflicting_self_doi_is_rejected(self):
+        html='<meta name="citation_title" content="Paper"><meta name="citation_doi" content="10.1007/different">'
+        with self.assertRaisesRegex(ValueError,'DOI differs'):
+            parse_paper(html,self.entry(),'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php')
+
+    def test_legacy_ecva_author_formats_do_not_become_extra_people(self):
+        entry=dict(self.entry(),index_authors='Doe, Jane and Smith, John')
+        page='<div id="papertitle">Paper</div><div id="authors">Jane Doe, John Smith; The European Conference on Computer Vision (ECCV), 2018, pp. 1-12</div>'
+        row=parse_paper(page,entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php')
+        self.assertEqual(row['authors'],['Jane Doe','John Smith'])
+        row=parse_paper('<div id="papertitle">Paper</div>',entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php')
+        self.assertEqual(row['authors'],['Jane Doe','John Smith'])
+
+    def test_cvpr_day_one_is_replaced_by_explicit_all_papers_index(self):
+        root='https://openaccess.thecvf.com/CVPR2026';all_url=root+'?day=all'
+        first='https://openaccess.thecvf.com/content/CVPR2026/html/First_CVPR_2026_paper.html'
+        second='https://openaccess.thecvf.com/content/CVPR2026/html/Second_CVPR_2026_paper.html'
+        def item(url):return f'<dt class="ptitle"><a href="{url}">Paper</a></dt><dd>Jane Doe, John Smith</dd><dd>[<a href="never-download.pdf">pdf</a>]</dd>'
+        pages={root:f'<a href="?day=all">All papers</a><dl>{item(first)}</dl>',all_url:f'<dl>{item(first)}{item(second)}</dl>',first:'<div id="papertitle">First paper</div><div id="abstract">First abstract</div>',second:'<div id="papertitle">Second paper</div><p>Footer without an abstract</p>'}
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
+            manifest=collect_index('cvf',{'venue':'CVPR','year':2026,'track':'main','index_url':root},tmp,tmp)
+        self.assertEqual(manifest['count'],2)
+        self.assertEqual(manifest['list_url'],all_url)
+        self.assertEqual(manifest['status'],'partial')
+        self.assertEqual(manifest['missing_abstract_count'],1)
+        self.assertEqual({call.args[0] for call in fetch.call_args_list},{root,all_url,first,second})
+
+    def test_discovery_keeps_findings_separate_and_excludes_workshops(self):
+        html='<a href="/CVPR2026">Main</a><a href="/CVPR2026_findings">Findings</a><a href="/ICCV2025">Main</a><a href="/CVPR2013.py">Legacy main</a><a href="/CVPR2026_workshops/menu">Workshops</a><a href="/WACV2026">WACV</a>'
+        with patch('paper_atlas.collectors.vision.fetch',return_value=html.encode()):indexes=discover('cvf','unused')
+        self.assertEqual({(x['venue'],x['year'],x['track']) for x in indexes},{('CVPR',2013,'main'),('CVPR',2026,'main'),('CVPR',2026,'findings'),('ICCV',2025,'main')})
+
+    def test_unparsed_publication_link_does_not_silently_reduce_coverage(self):
+        html='<a href="content/CVPR2026/html/Unparsed_CVPR_2026_paper.html">Other paper</a><dl><dt><a href="content/CVPR2026/html/First_CVPR_2026_paper.html">Paper</a></dt></dl>'
+        with self.assertRaisesRegex(ValueError,'outside parsed'):
+            index_entries(html,'https://openaccess.thecvf.com/')
+
+    def test_legacy_declared_encoding_is_respected(self):
+        html='<meta content="text/html; charset=ISO-8859-1"><div>André</div>'
+        self.assertIn('André',decode_html(html.encode('iso-8859-1')))
+
+
+if __name__=='__main__':unittest.main()