Compare commits

...

21 Commits

Author SHA1 Message Date
codex-bot b1429cd3d8 Preserve the CoRL 2021 Blue Sky publication track
Metadata integrity checks / metadata (push) Successful in 10m39s
2026-11-22 19:31:39 +08:00
Zayden-Jung 2988e4e42b Merge pull request '提供 PostgreSQL 全库身份核验与恢复流程' (#89) from codex/global-production-audit into main
Metadata integrity checks / metadata (push) Successful in 10m28s
Reviewed-on: #89
2026-11-22 07:04:46 +00:00
codex-bot 4c834dc10e Add a reproducible PostgreSQL global identity audit
Metadata integrity checks / metadata (push) Successful in 8m22s
2026-11-22 15:02:25 +08:00
Zayden-Jung b535cc556b Merge pull request '压缩大型出版来源审计计划,避免 PostgreSQL 写入内存峰值' (#88) from codex/source-revision-storage into main
Metadata integrity checks / metadata (push) Successful in 7m44s
Reviewed-on: #88
2026-11-21 08:55:30 +00:00
codex-bot dc4f61d2f6 Compress large source revision snapshots before PostgreSQL writes
Metadata integrity checks / metadata (push) Successful in 7m33s
2026-11-20 23:20:26 +08:00
Zayden-Jung 04b2ed9e95 Merge pull request '核验 CLeaR 五届正式出版与公开拒稿来源' (#87) from codex/clear-coverage into main
Metadata integrity checks / metadata (push) Successful in 9m26s
Reviewed-on: #87
2026-11-20 14:55:16 +00:00
codex-bot 8c6a328a4d Record verified CLeaR annual programme links
Metadata integrity checks / metadata (push) Successful in 9m45s
2026-11-20 21:35:58 +08:00
codex-bot 710d84a8bf Document complete CLeaR dataset track roster
Metadata integrity checks / metadata (push) Successful in 9m16s
2026-11-20 21:17:20 +08:00
codex-bot d138981da4 Record CLeaR publication and negative-source coverage
Metadata integrity checks / metadata (push) Successful in 8m27s
2026-11-20 21:03:40 +08:00
Zayden-Jung ef8c1cfeb9 Merge pull request '补齐公开决定来源与 ICASSP 2026 赛道核验' (#86) from codex/public-decision-source-boundaries into main
Metadata integrity checks / metadata (push) Successful in 15m17s
Reviewed-on: #86
2026-11-20 13:01:48 +00:00
Zayden-Jung 5e5d07d086 Merge pull request '封装 KDD 2026 两周期目录与撤回标注' (#85) from codex/kdd2026-source-cycles into main
Metadata integrity checks / metadata (push) Successful in 9m23s
Reviewed-on: #85
2026-11-19 09:03:43 +00:00
Zayden-Jung 50ffc2d4b6 Merge pull request '给已有 ECCV 2026 投稿补挂正式出版证据' (#84) from codex/eccv-publication-reconciliation into main
Metadata integrity checks / metadata (push) Successful in 7m27s
Reviewed-on: #84
2026-11-19 09:03:31 +00:00
Zayden-Jung e454ed51ec Merge pull request '记录六个会议公开决定来源的范围与待核项' (#83) from codex/public-decision-source-boundaries into main
Metadata integrity checks / metadata (push) Successful in 8m29s
Reviewed-on: #83
2026-11-19 09:03:25 +00:00
Zayden-Jung 32a53977ee Merge pull request '保留 Springer 章节中经核验的单名作者' (#82) from codex/springer-single-author-name into main
Metadata integrity checks / metadata (push) Successful in 8m52s
Reviewed-on: #82
2026-11-19 09:03:20 +00:00
Zayden-Jung da5b10f2a6 Merge pull request '兼容 ECCV 2026 Springer 标题的数字上标格式' (#81) from codex/eccv2026-publisher-title-formatting into main
Metadata integrity checks / metadata (push) Successful in 8m19s
Reviewed-on: #81
2026-11-19 09:03:14 +00:00
codex-bot 6e5202c81a Inventory KDD 2026 programme cycles and withdrawal evidence
Metadata integrity checks / metadata (push) Successful in 8m7s
2026-11-18 23:14:36 +08:00
codex-bot ae8bcd4c14 Close the SQLite fixture before PostgreSQL migration
Metadata integrity checks / metadata (push) Successful in 9m49s
2026-11-18 22:35:16 +08:00
codex-bot 3acb738aa8 Supply authors in the ECCV integration test seed
Metadata integrity checks / metadata (push) Failing after 6m32s
2026-11-18 22:27:05 +08:00
codex-bot cbf7bab8ca Attach ECCV publication evidence to existing programme attempts
Metadata integrity checks / metadata (push) Failing after 6m0s
2026-11-18 22:19:43 +08:00
codex-bot 4d97822d9e Preserve confirmed single-name Springer authors
Metadata integrity checks / metadata (push) Successful in 8m28s
2026-11-18 20:30:11 +08:00
codex-bot bc01f90e70 Preserve Springer title literals across superscript formatting
Metadata integrity checks / metadata (push) Successful in 12m44s
2026-11-18 19:25:00 +08:00
28 changed files with 9165 additions and 31 deletions
+14
View File
@@ -47,6 +47,20 @@ jobs:
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True) subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
outside = root / 'outside-checkout' outside = root / 'outside-checkout'
outside.mkdir() outside.mkdir()
kdd_snapshot = pathlib.Path('/data/paper-index/validation/kdd2026-official-papers-html-v1.html')
if kdd_snapshot.is_file():
kdd_output = root / 'installed-kdd2026.json'
subprocess.run([python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'kdd2026-program',
'--snapshot', str(kdd_snapshot), '--snapshot-sha256',
'47ee82841dedf034614348d7fdb29714e8ad62fca8c6811e1d856845588f145c',
'--out', str(kdd_output)], cwd=outside, check=True)
kdd = json.loads(kdd_output.read_bytes())
assert kdd['unique_dois'] == 1408 and kdd['identical_repeat_rows'] == 7
assert {key:sum(counts.values()) for key,counts in kdd['source_rows_by_cycle_and_track'].items()} == {
'july':256, 'february':1159}
held, = [entry for entry in kdd['records'] if entry['programme_status'] != 'listed']
assert held['doi'] == '10.1145/3770855.3818478' and held['source_withdrawal_date_literal'] == '6/24/26'
print('KDD 2026 official cycle and withdrawal fixture passed', flush=True)
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1') cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
if cikm_snapshots.is_dir(): if cikm_snapshots.is_dir():
cikm_output = root / 'installed-cikm2026' cikm_output = root / 'installed-cikm2026'
+6 -2
View File
@@ -6,7 +6,7 @@
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。 Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。 历史目录包含 43 个会议。主动维护以核心 AI 会议及其已公开的决定为主,继续跟踪 ACM MM 和 MLSys。此前入库的 CIKM 数据与证据保留,后续主动补漏范围排除 CIKM。实际覆盖以数据库和来源清单为准。
当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。 当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。
@@ -108,6 +108,8 @@ AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)支持已核验的
CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤,不因代码测试通过而自动完成。 CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤,不因代码测试通过而自动完成。
一批数据全部验收后,运行 [`audit global-identity`](docs/operations.md#导入后的全库身份核验) 核对 PostgreSQL 的稳定标识、同题与近似标题候选、摘要完整性和全文检索。审计报告和候选文件对应一次快照,后续导入完成后重新生成。使用 `audit histories` 检查跨会议投稿决定及其来源关系。
开发通过工作分支和 PR 交付,由用户手动合并。 开发通过工作分支和 PR 交付,由用户手动合并。
早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。 早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。
@@ -147,8 +149,10 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。 `collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
[KDD 2026 官方目录](docs/kdd2026-program.md)可以按 DOI 核对两个投稿周期、分轨与目录中的撤回更换标注。
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。 `collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。 `collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。 ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。同届节目投稿可按独立审核计划附加出版证据,保留原投稿 ID 和既往决定。
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,35 @@
# CLeaR 2022–2026 逐篇覆盖核验
2026 年 11 月 20 日重新读取五卷 PMLR 目录,目录哈希与此前保存的出版目录核查一致。逐个出版 ID 对照 PostgreSQL。243 条全部关联到对应会年的主会录用。每个出版 ID 有一条录用关联,243 条来源记录均有摘要。 [逐条回执](clear-2022-2026-record-coverage.json)
| 年份 | 目录 | 录用关联 |
| --- | --- | --- |
| 2022 | [PMLR 177](https://proceedings.mlr.press/v177/) | 48/48 |
| 2023 | [PMLR 213](https://proceedings.mlr.press/v213/) | 39/39 |
| 2024 | [PMLR 236](https://proceedings.mlr.press/v236/) | 45/45 |
| 2025 | [PMLR 275](https://proceedings.mlr.press/v275/) | 58/58 |
| 2026 | [PMLR 323](https://proceedings.mlr.press/v323/) | 53/53 |
2026 年投稿规则允许录用作者用一页扩展摘要代替全文发表。核验覆盖上述正式出版目录。节目单上可能另有展示记录,需按主办方的逐篇证据单独核对。 [2026 官方征稿](https://www.cclear.cc/2026/CallforPapers)
五届公开 OpenReview 分组均可读取。2022–2023 分组未提供本次查询所需的拒稿赛道字段。2024–2026 分组将撤回、初筛拒稿默认公开标记设为 false,并提供拒稿赛道标识。对 2024 拒稿赛道的第一条匿名请求返回 HTTP 403,随后停止。2025–2026 的具体公开拒稿可见性因此仍待核。五届当前库内无逐篇负面决定。本次为只读核验。 [OpenReview 会议目录](https://openreview.net/venue?id=cclear.cc%2FCLeaR) · [逐年核验回执](clear-2022-2026-record-coverage.json)
## 2023 数据集赛道
主办方另外举行了因果基准数据集赛道,以邮件收件,并公开了 6 条录用。六条均已入库并通过常规查询回查。它们单列在 `dataset` 赛道,原主会 39 条录用关联通过写入前后复验。 [官方征集](https://www.cclear.cc/2023/CallforDatasets) · [官方录用名单](https://www.cclear.cc/2023/AcceptedDatasets) · [逐条查询回执](clear-2023-dataset-roster.json)
SCARY 关联到同题同作者的 arXiv 作品。SpaCE 的共同作者明确将 CLeaR 2023 列为早期版本,已关联到同一作品并保留 ICLR 2024 的录用历史和两个版本的署名。另 4 篇按官方名单暂定建档,近似题名查询超时,仍需最终去重复核。六条名单来源记录的摘要留空。名单仅公布录用,本次未新增负面决定。 [SCARY 预印本](https://arxiv.org/abs/2304.14109) · [SpaCE 共同作者说明](https://anatrisovic.com/docs/atrisovic-cv.pdf)
## 主会录用名单和日程
以主办方逐篇名单和日程补挂展示来源。原投稿及决定继续使用原 ID。展示形式以来源标签保存,同篇可同时有口头报告和海报。 [逐篇回执](clear-2022-2026-main-programme.json)
| 年份 | 已关联作品 | 来源及展示情况 |
| --- | ---: | --- |
| 2022 | 49 | 9 条口头报告、40 条海报。另于 PMLR 48 条之外补入一条扩展摘要录用。 |
| 2023 | 39 | 8 条口头报告、31 条海报。一个节目题名变体由作者技术报告核对。 |
| 2024 | 45 | 59 项日程展示,含同篇的多项展示。5 条日程题名变体按正式出版页和作者资料核对。 |
| 2025 | 58 | 9 条口头报告和 58 条海报展示,同篇共享投稿。 |
| 2026 | 9 | 9 条日程明列题名的口头报告。海报只列投稿编号,逐篇对应仍待核。 |
2024 与 2025 的网站录用页指向 2023 名单,使用各年完整版日程。2026 数据集赛道征稿规定筛选后在独立网站发布并开放投票,逐篇结果名单仍待定位。 [2024 日程](https://www.cclear.cc/2024/FullAgenda) · [2025 日程](https://www.cclear.cc/2025/FullAgenda) · [2026 日程](https://www.cclear.cc/2026/FullAgenda) · [2026 数据集征集](https://www.cclear.cc/2026/CallforDatasets)
@@ -0,0 +1,178 @@
{
"status": "committed_and_readonly_verified",
"snapshot_at": "2026-11-20 13:14:53.43621+00",
"roster_status": "complete",
"expected": 6,
"imported": 6,
"new": 5,
"records": [
{
"external_id": "bizeul23a",
"title": "3DIdentBox: A Toolbox for Identifiability Benchmarking",
"authors_json": "[\"Alice Bizeul\", \"Imant Daunhawer\", \"Emanuele Palumbo\", \"Bernhard Schölkopf\", \"Alexander Marx\", \"Julia E. Vogt\"]",
"paper_id": 3572631,
"source_record_id": 3824788,
"submission_id": 541329,
"decision_id": 580918,
"status": "accepted",
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
"decision_date": null
},
{
"external_id": "chen23a",
"title": "The Structurally Complex with Additive Parent Causality (SCARY) Dataset",
"authors_json": "[\"Jarry Chen\", \"Haytham M. Fayek\"]",
"paper_id": 1834131,
"source_record_id": 3824787,
"submission_id": 541328,
"decision_id": 580917,
"status": "accepted",
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
"decision_date": null
},
{
"external_id": "gong23a",
"title": "CausalEdu: a real-world education dataset for temporal causal discovery and inference",
"authors_json": "[\"Wenbo Gong\", \"Digory Smith\", \"Zichao Wang\", \"Craig Barton\", \"Simon Woodhead\", \"Nick Pawlowski\", \"Joel Jennings\", \"Cheng Zhang\"]",
"paper_id": 3572634,
"source_record_id": 3824792,
"submission_id": 541333,
"decision_id": 580922,
"status": "accepted",
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
"decision_date": null
},
{
"external_id": "munozbenitez23a",
"title": "Synthetic Time Series: A Dataset for Causal Discovery",
"authors_json": "[\"Julio Munoz-Benıtez\", \"L. Enrique Sucar\"]",
"paper_id": 3572633,
"source_record_id": 3824790,
"submission_id": 541331,
"decision_id": 580920,
"status": "accepted",
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
"decision_date": null
},
{
"external_id": "tec23a",
"title": "SpaCE: The Spatial Confounding (Benchmarking) Environment",
"authors_json": "[\"Mauricio Tec\", \"Ana Trisovic\", \"Michelle Audirac\", \"Francesca Dominici\"]",
"paper_id": 1964409,
"source_record_id": 3824791,
"submission_id": 541332,
"decision_id": 580921,
"status": "accepted",
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
"decision_date": null
},
{
"external_id": "xu23a",
"title": "Causal Benchmark Based on Disentangled Image Dataset",
"authors_json": "[\"Liyuan Xu\", \"Arthur Gretton\"]",
"paper_id": 3572632,
"source_record_id": 3824789,
"submission_id": 541330,
"decision_id": 580919,
"status": "accepted",
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
"decision_date": null
}
],
"source": "clear2023_dataset_roster",
"edition_id": 104179,
"roster": "https://www.cclear.cc/2023/AcceptedDatasets",
"bundle_sha256": "d5b9a0cda61581da679b1de7180da74dcc4d70625c984a401b10820083719a76",
"original_main_bindings_preserved": 39,
"prior_space_links_preserved": 2,
"roster_entries": [
{
"entry_id": "chen23a",
"title": "The Structurally Complex with Additive Parent Causality (SCARY) Dataset",
"authors": [
"Jarry Chen",
"Haytham M. Fayek"
],
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/chen23a.pdf"
},
{
"entry_id": "bizeul23a",
"title": "3DIdentBox: A Toolbox for Identifiability Benchmarking",
"authors": [
"Alice Bizeul",
"Imant Daunhawer",
"Emanuele Palumbo",
"Bernhard Schölkopf",
"Alexander Marx",
"Julia E. Vogt"
],
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/bizeul23a.pdf"
},
{
"entry_id": "xu23a",
"title": "Causal Benchmark Based on Disentangled Image Dataset",
"authors": [
"Liyuan Xu",
"Arthur Gretton"
],
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/xu23a.pdf"
},
{
"entry_id": "munozbenitez23a",
"title": "Synthetic Time Series: A Dataset for Causal Discovery",
"authors": [
"Julio Munoz-Benıtez",
"L. Enrique Sucar"
],
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/munozbenitez23a.pdf"
},
{
"entry_id": "tec23a",
"title": "SpaCE: The Spatial Confounding (Benchmarking) Environment",
"authors": [
"Mauricio Tec",
"Ana Trisovic",
"Michelle Audirac",
"Francesca Dominici"
],
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/tec23a.pdf"
},
{
"entry_id": "gong23a",
"title": "CausalEdu: a real-world education dataset for temporal causal discovery and inference",
"authors": [
"Wenbo Gong",
"Digory Smith",
"Zichao Wang",
"Craig Barton",
"Simon Woodhead",
"Nick Pawlowski",
"Joel Jennings",
"Cheng Zhang"
],
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/gong23a.pdf"
}
],
"roster_bundle_url": "https://www.cclear.cc/static/js/main.901abbee.chunk.js",
"normal_query_verification": {
"standard_query_verified": [
580918,
580917,
580922,
580920,
580921,
580919
],
"roster_count": 6
},
"space_identity_author_source": "https://anatrisovic.com/docs/atrisovic-cv.pdf",
"provisional_work_ids": [
3572631,
3572632,
3572633,
3572634
],
"abstract_coverage": "The six source records have no roster-supplied abstracts. Existing canonical abstracts of linked works are retained.",
"public_negative_decision_coverage": "The official accepted dataset roster does not publish a negative submission list. No negative decision was imported.",
"deduplication_status": "Four provisional works await cross-source near-title review. The previous unbounded name query ended on timeout without results."
}
+17 -2
View File
@@ -9,8 +9,23 @@ paper-atlas collect eccv2026-publications \
--out /data/paper-index/collected/eccv-publisher-metadata --out /data/paper-index/collected/eccv-publisher-metadata
``` ```
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。 目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。数字上标的两种数学定界符可按同一标题核对,章节标题和目录原文分别保存。已有输出目录需另查回执或换新空目录后运行。
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。 输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。单名署名须由同页的结构化作者列表按顺序确认,确认后按原名保存为字符串,引用元数据仍保留原文。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。 生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
## 已有节目投稿的出版证据
先在只读快照上复核节目 ID、章节 DOI、标题、完整有序作者、摘要版本与当前投稿历史。保存计划和原始证据,计划应列出每条出版观察、保留的论文与投稿 ID、节目来源和决定 ID、当前与目标届次及完整历史包的 SHA-256。作者或版本仍有疑点的条目继续留在待核清单。
```sh
paper-atlas maintain eccv-publication-reconciliation \
--pg-service <writer-service> --pg-schema research \
--plan <reviewed-plan.json> --plan-sha256 <reviewed-plan-sha256> \
--springer-previous-count <last-verified-count>
```
命令接收 `readonly_same_submission_reconciliation_plan` 格式的已复核计划。它会在同一事务内核对实时历史和来源覆盖,把原节目投稿归入主会,追加出版来源与观测决定,并分别调整节目与 Springer 的覆盖数。节目主会覆盖保持 partial,因为其目录的主会分母尚未独立核定。随后通过只读连接核验投稿、决定及覆盖,保存计划和回执。相同计划只允许成功提交一次。若执行中失败,先查同一计划的提交标记、覆盖和投稿来源,再决定是否重新执行。
旧节目快照的全量未分类 manifest 无法刷新已有逐条更正的投稿。继续刷新时需按已复核轨道分别整理输出并生成 manifest。
+18
View File
@@ -0,0 +1,18 @@
# KDD 2026 官方论文目录
从 [KDD 2026 论文目录](https://kdd2026.kdd.org/papers/) 保存公开 HTML,记录其 SHA-256,在已有目录中指定新的输出文件。
```sh
paper-atlas collect kdd2026-program \
--snapshot kdd2026-papers.html \
--snapshot-sha256 <snapshot-sha256> \
--out kdd2026-programme.json
```
采集器只读取已保存的页面数据,逐项输出 DOI、官网标题、投稿周期、轨道和行哈希。完全相同的重复条目保留出现次数,冲突记录中止解析并等待人工复核。输出用于与来源和库内 DOI 核对,不能直接交给记录导入命令。
已核对的官网快照有 1,415 条目录记录和 1,408 个唯一 DOI。7 条重复记录的周期、轨道、标题和作者文本一致。February Cycle 列出 Research、Applied Data Science、Datasets and Benchmarks、AI for Sciences、Blue Sky Ideas 五轨。July Cycle 列出前三轨。官网列出的演讲、海报或扩展摘要需要出版方记录确认归档状态。[AI for Sciences 投稿及出版规则](https://kdd2026.kdd.org/ai4sciences-track-call-for-papers/)
Applied Data Science 有一条官网标注撤回并更换投稿 ID 的记录,旧链接为 `10.1145/3770855.3818478`。保留官网原文和日期字面值。原始投稿 ID 仍待公开证据核实。arXiv 当前指向 `10.1145/3770855.3828560`,这个 DOI 已属于同一份已有作品。两份来源分别记录,旧链接不作为新增作品标识。[arXiv 当前记录](https://arxiv.org/abs/2606.08923)
补分轨时按 DOI 检查库内已有投稿,先处理重复或缺少作者的记录。官网对录用材料的公开承诺适用于各轨道规定的范围,逐篇拒稿、初筛拒稿和其他撤回仍需公开决定证据。[Research](https://kdd2026.kdd.org/research-track-call-for-papers/) · [Applied Data Science](https://kdd2026.kdd.org/applied-data-science-ads-track-call-for-papers/) · [Datasets and Benchmarks](https://kdd2026.kdd.org/datasets-and-benchmarks-track-call-for-papers/) · [Blue Sky Ideas](https://kdd2026.kdd.org/kdd-2026-blue-sky-ideas-track-call-for-papers/)
+16
View File
@@ -1,5 +1,21 @@
# 运行维护与数据核验 # 运行维护与数据核验
## 导入后的全库身份核验
在服务器准备一个至少有 2 GiB 可用空间的审计目录。使用生产读取角色,在计划中的导入全部验收后运行以下命令。命令在 PostgreSQL 的同一个可重复读快照内检查库内记录。SQLite 临时文件包含用来比较的标题键与作者指纹,扫描结束后清理。
```bash
paper-atlas audit global-identity \
--pg-service foundry-research --pg-schema research \
--scratch-root /data/paper-index/validation \
--candidate-output /data/paper-index/validation/identity-candidates-2026-11-22.jsonl \
> /data/paper-index/validation/identity-report-2026-11-22.json
```
JSON 报告记录快照时间、论文与缺摘要计数、稳定标识冲突、约束状态、会议年度与轨道的来源覆盖,以及一次实际的全文搜索和执行计划。JSONL 保存同题且规范化作者集合相同的论文组,以及相同作者集合中标题相似度至少为 0.90 的论文对。超过 100 条记录的作者组会列入报告,留作后续逐组核对。缺作者、作者增减或换名的版本需要结合 DOI、论坛或作者说明,使用 `audit source-identities` 和原始来源复核。
将报告、候选文件、运行版本与 SHA-256 一起保留。使用 `audit histories` 核对拒稿、撤稿、后续录用和来源链接。需要合并候选时先确认论文版本关系并保存投稿、决定、标签及来源证据,再按已授权的流程操作。新一批导入完成后更新快照和审计报告。
## PostgreSQL 迁移 ## PostgreSQL 迁移
共享实例的容器名为 `foundry-postgres`,底层镜像为 `postgres:17-alpine`;Compose 服务名仍为 `postgres`。数据库和 schema 是可配置的逻辑边界,表按研究数据域组织,不按调用项目复制。其他项目可获授 `research` 的 USAGE 及必要表权限,复用相同论文 ID 和表。未来换库只更新 libpq service;换 schema 可更新角色 search_path 或查询的 `--pg-schema`。 共享实例的容器名为 `foundry-postgres`,底层镜像为 `postgres:17-alpine`;Compose 服务名仍为 `postgres`。数据库和 schema 是可配置的逻辑边界,表按研究数据域组织,不按调用项目复制。其他项目可获授 `research` 的 USAGE 及必要表权限,复用相同论文 ID 和表。未来换库只更新 libpq service;换 schema 可更新角色 search_path 或查询的 `--pg-schema`。
+2
View File
@@ -48,6 +48,8 @@ paper-atlas maintain source-revisions \
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。 同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
源快照和计划各自在 UTF-8 JSON 达到 1 MiB 时由客户端压缩后写入,避免数据库在接收大参数时出现内存峰值。读取兼容既有纯 JSON 和带 `gzip+base64:v1:` 前缀的压缩值,完整内容和计划哈希均可按原口径核验。使用该表执行写入或刷新时,运行时需要支持两种存储格式。
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。 提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
## 后续刷新及限制 ## 后续刷新及限制
+199
View File
@@ -0,0 +1,199 @@
"""Read-only PostgreSQL identity and integrity audit for an existing snapshot."""
import argparse
from collections import Counter
from contextlib import closing
from difflib import SequenceMatcher
from hashlib import sha256
from itertools import combinations, groupby
import json
from pathlib import Path
import shutil
import sqlite3
from tempfile import TemporaryDirectory
from paper_atlas.db.store import author_keys
from paper_atlas.postgres.client import ReadConnection, SEARCH_VECTOR
ROSTER_GROUP_LIMIT = 100
TITLE_SCORE = .90
def _scalar(connection, query, params=()):
return connection.execute(query, params).fetchone()[0]
def _checks(connection):
counts = {}
for table, columns in (
('identifiers', 'scheme,value'),
('source_records', 'source_id,external_id'),
('submission_identifiers', 'edition_id,scheme,value'),
):
counts[f'duplicate_{table}_keys'] = _scalar(connection, f'''
SELECT count(*) FROM (
SELECT {columns} FROM {table} GROUP BY {columns} HAVING count(*)>1
) duplicates''')
counts['normalized_doi_collisions'] = _scalar(connection, '''
SELECT count(*) FROM (
SELECT lower(regexp_replace(btrim(value),
?, '', 'i')) AS doi
FROM identifiers WHERE lower(scheme)='doi' GROUP BY 1
HAVING count(DISTINCT paper_id)>1
) duplicates''', (r'^(https?://(dx\.)?doi\.org/|doi:)[[:space:]]*',))
counts['multiple_papers_per_base_arxiv_id'] = _scalar(connection, r'''
SELECT count(*) FROM (
SELECT regexp_replace(btrim(value),'v[0-9]+$','','i') AS arxiv_id
FROM identifiers WHERE lower(scheme)='arxiv' GROUP BY 1
HAVING count(DISTINCT paper_id)>1
) duplicates''')
counts['submission_identifier_wrong_edition'] = _scalar(connection, '''
SELECT count(*) FROM submission_identifiers i
JOIN submissions s ON s.id=i.submission_id WHERE s.edition_id<>i.edition_id''')
counts['unvalidated_foreign_key_or_check_constraints'] = _scalar(connection, '''
SELECT count(*) FROM pg_constraint k JOIN pg_namespace n ON n.oid=k.connamespace
WHERE n.nspname=current_schema() AND k.contype IN ('f','c') AND NOT k.convalidated''')
counts['tag_wrong_source_paper'] = _scalar(connection, '''
SELECT count(*) FROM paper_tags t JOIN source_records r ON r.id=t.source_record_id
WHERE t.paper_id<>r.paper_id''')
return counts
def _coverage(connection):
return [dict(row) for row in connection.execute('''
SELECT v.acronym AS venue,e.year,e.track,e.event_key,o.name AS source,
l.list_url,l.status,l.expected_count,l.imported_count
FROM edition_sources l JOIN editions e ON e.id=l.edition_id
JOIN venues v ON v.id=e.venue_id JOIN sources o ON o.id=l.source_id
ORDER BY v.acronym,e.year,e.track,e.event_key,o.name,l.list_url''')]
def _fts(connection):
indexed = bool(_scalar(connection, '''
SELECT count(*) FROM pg_index i JOIN pg_class t ON t.oid=i.indrelid
JOIN pg_class x ON x.oid=i.indexrelid JOIN pg_am a ON a.oid=x.relam
JOIN pg_namespace n ON n.oid=t.relnamespace
WHERE n.nspname=current_schema() AND t.relname='papers' AND a.amname='gin'
AND i.indisvalid AND i.indisready AND x.relname='papers_search_gin'
AND strpos(pg_get_indexdef(i.indexrelid),'to_tsvector')>0'''))
query = 'SELECT id FROM papers WHERE ' + SEARCH_VECTOR + " @@ websearch_to_tsquery('english',?) LIMIT 5"
phrase = 'convolutional vision transformer'
plan = [row[0] for row in connection.execute('EXPLAIN (COSTS OFF) '+query, (phrase,))]
ids = [row[0] for row in connection.execute(query, (phrase,))]
return {'gin_index_ready': indexed, 'plan': plan, 'matched_paper_ids': ids, 'query': phrase}
def _author_fingerprint(raw):
authors = sorted(author_keys(json.loads(raw)))
if not authors:
return None
return sha256(json.dumps(authors, ensure_ascii=False).encode()).digest()
def _stage_titles(connection, scratch):
scratch.execute('CREATE TABLE titles (id INTEGER PRIMARY KEY, title_key TEXT NOT NULL, authors BLOB)')
counts = Counter(papers=0, missing_abstract=0, missing_author_keys=0, invalid_authors_json=0)
with connection.connection.cursor(name='paper_atlas_global_identity_scan') as titles:
titles.itersize = 5000
titles.execute('SELECT id,title_key,authors_json,abstract IS NULL OR btrim(abstract)=\'\' FROM papers ORDER BY id')
while rows := titles.fetchmany(5000):
batch = []
for row in rows:
paper_id,title_key,authors,missing = (row[index] for index in range(4))
try:
fingerprint = _author_fingerprint(authors)
except (TypeError, ValueError):
fingerprint = None
counts['invalid_authors_json'] += 1
counts['papers'] += 1
counts['missing_abstract'] += int(missing)
counts['missing_author_keys'] += int(fingerprint is None)
batch.append((paper_id,title_key,fingerprint))
scratch.executemany('INSERT INTO titles VALUES (?,?,?)', batch)
scratch.commit()
scratch.execute('CREATE INDEX keyed_titles ON titles (title_key,authors,id)')
scratch.execute('CREATE INDEX title_rosters ON titles (authors,title_key,id) WHERE authors IS NOT NULL')
return dict(counts)
def _candidates(scratch, output):
counts = Counter(exact_title_groups=0, exact_title_and_roster_groups=0,
exact_title_and_roster_papers=0, near_title_and_roster_pairs=0,
oversized_rosters=0, papers_in_oversized_rosters=0)
def emit(data):
output.write(json.dumps(data, ensure_ascii=False, separators=(',',':'))+'\n')
counts['exact_title_groups'] = scratch.execute('''
SELECT count(*) FROM (SELECT title_key FROM titles WHERE title_key<>''
GROUP BY title_key HAVING count(*)>1)''').fetchone()[0]
for title,roster,size in scratch.execute('''
SELECT title_key,authors,count(*) FROM titles WHERE title_key<>'' AND authors IS NOT NULL
GROUP BY title_key,authors HAVING count(*)>1 ORDER BY title_key,authors'''):
counts['exact_title_and_roster_groups'] += 1
counts['exact_title_and_roster_papers'] += size
ids = [row[0] for row in scratch.execute('''
SELECT id FROM titles WHERE title_key=? AND authors=? ORDER BY id''', (title,roster))]
emit({'kind':'exact_title_and_roster','title_key':title,'paper_ids':ids})
cursor = scratch.execute('''
SELECT authors,title_key,id FROM titles WHERE authors IS NOT NULL AND length(title_key)>=25
ORDER BY authors,title_key,id''')
for _, group in groupby(cursor, key=lambda row:row[0]):
entries = []
size = 0
for _, title, paper_id in group:
size += 1
if size <= ROSTER_GROUP_LIMIT:
entries.append((title,paper_id))
if size > ROSTER_GROUP_LIMIT:
counts['oversized_rosters'] += 1
counts['papers_in_oversized_rosters'] += size
emit({'kind':'oversized_roster','size':size,'sample_paper_ids':[p for _,p in entries[:5]]})
continue
for (left,left_id),(right,right_id) in combinations(entries,2):
if left==right or 2*min(len(left),len(right))/(len(left)+len(right)) < TITLE_SCORE:
continue
matcher = SequenceMatcher(None,left,right,autojunk=False)
if matcher.quick_ratio()>=TITLE_SCORE and (score:=matcher.ratio())>=TITLE_SCORE:
counts['near_title_and_roster_pairs'] += 1
emit({'kind':'near_title_and_roster','paper_ids':[left_id,right_id],'score':round(score,5)})
return dict(counts)
def report(connection, scratch_root, candidates):
transaction = dict(connection.execute('''
SELECT current_database() AS database,current_schema() AS schema,
current_setting('transaction_read_only') AS read_only,
current_setting('transaction_isolation') AS isolation,
transaction_timestamp()::text AS snapshot_at''').fetchone())
if transaction['read_only']!='on' or transaction['isolation']!='repeatable read':
raise ValueError('A read-only repeatable-read snapshot is required')
if shutil.disk_usage(scratch_root).free < 2*1024**3:
raise ValueError('The scratch filesystem must have at least 2 GiB free')
connection.execute("SET LOCAL statement_timeout='180s'")
integrity = _checks(connection)
coverage = _coverage(connection)
search = _fts(connection)
with TemporaryDirectory(prefix='paper-atlas-audit-',dir=scratch_root) as directory:
with closing(sqlite3.connect(Path(directory)/'titles.sqlite')) as scratch:
scratch.execute('PRAGMA temp_store=FILE')
counts = _stage_titles(connection,scratch)
candidate_counts = _candidates(scratch,candidates)
return {'snapshot':transaction,'papers':counts,'integrity':integrity,
'candidate_counts':candidate_counts,'edition_sources':coverage,'search':search,
'candidate_rules':{'minimum_normalized_title_length':25,'near_title_score':TITLE_SCORE,
'max_papers_per_near_title_author_group':ROSTER_GROUP_LIMIT,
'requires_exact_normalized_author_set':True},
'canonical_data_changed':False}
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog,description='只读核验 PostgreSQL 全库身份候选、约束、来源覆盖和全文检索')
parser.add_argument('--pg-service',required=True)
parser.add_argument('--pg-schema')
parser.add_argument('--scratch-root',type=Path,required=True)
parser.add_argument('--candidate-output',type=Path,required=True)
args = parser.parse_args(argv)
with closing(ReadConnection(args.pg_service,args.pg_schema)) as connection:
with args.candidate_output.open('x',encoding='utf-8') as candidates:
result = report(connection,args.scratch_root,candidates)
print(json.dumps(result,ensure_ascii=False))
+20 -14
View File
@@ -173,34 +173,40 @@ def find_candidates(connection, records):
records[hit['n']]['title_candidate_ids'].append(hit['paper_id']) records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
def _named_rows(connection, sql, params):
cursor = connection.execute(sql, params)
columns = [field[0] for field in cursor.description]
return [{name: row[i] for i, name in enumerate(columns)} for row in cursor]
def entity_packet(connection, paper_id): def entity_packet(connection, paper_id):
paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone() papers = _named_rows(connection, 'SELECT * FROM papers WHERE id=?', (paper_id,))
if paper is None: if not papers:
raise RuntimeError('Candidate disappeared from read-only snapshot') raise RuntimeError('Candidate disappeared from read-only snapshot')
sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source sources = _named_rows(connection, '''SELECT r.*,s.name AS source
FROM source_records r JOIN sources s ON s.id=r.source_id FROM source_records r JOIN sources s ON s.id=r.source_id
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))] WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))
attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key attempts = _named_rows(connection, '''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))] WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))
packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute( packet = {'paper': papers[0], 'identifiers': _named_rows(connection,
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))], 'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,)),
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100, 'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200} 'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
for attempt in packet['attempts']: for attempt in packet['attempts']:
observations = [dict(r) for r in connection.execute( observations = _named_rows(connection,
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))] 'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200 attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source provenance = _named_rows(connection, '''SELECT ss.*,r.external_id,s.name AS source
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=? JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))] ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100 attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
for decision in attempt['decisions']: for decision in attempt['decisions']:
provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source provenance = _named_rows(connection, '''SELECT ds.*,r.external_id,s.name AS source
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=? JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))] ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100 decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
return packet return packet
+3
View File
@@ -35,6 +35,7 @@ COMMANDS = {
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'), ('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'), ('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
('collect','kdd2026-program'): ('paper_atlas.collectors.kdd2026', {}, '核对 KDD 2026 两个投稿周期、分轨及撤回更换标注'),
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'), ('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'), ('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
@@ -49,6 +50,7 @@ COMMANDS = {
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'), ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'), ('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
('maintain','eccv-publication-reconciliation'): ('paper_atlas.ingestion.publication_reconciliation', {}, '向既有 ECCV 2026 投稿附加已复核的正式出版证据'),
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'), ('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'), ('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'), ('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
@@ -64,6 +66,7 @@ COMMANDS = {
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'), ('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'), ('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'),
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'), ('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
('audit','global-identity'): ('paper_atlas.audits.global_identity', {}, '只读核对 PostgreSQL 身份候选与完整性'),
} }
@@ -102,10 +102,20 @@ def publisher_inventory(root):
return entries return entries
def title_for_comparison(title):
# Springer may render a numeric superscript with inline or display math delimiters.
return clean(re.sub(r'\\\((\^[0-9]+)\\\)|\$\$(\^[0-9]+)\$\$',
lambda match: match[1] or match[2], title))
def publisher_row(chapter, toc, evidence): def publisher_row(chapter, toc, evidence):
doi = chapter['doi'] doi = chapter['doi']
if clean(chapter['title']) != toc['title']: literal_title_differs = clean(chapter['title']) != toc['title']
if literal_title_differs and title_for_comparison(chapter['title']) != title_for_comparison(toc['title']):
raise ValueError('Chapter title differs from Springer contents, review the version') raise ValueError('Chapter title differs from Springer contents, review the version')
contents_evidence = {'url': toc['evidence_url'], 'sha256': toc['evidence_sha256']}
if literal_title_differs:
contents_evidence['title_literal'] = toc['title']
url = ORIGIN + '/chapter/' + doi url = ORIGIN + '/chapter/' + doi
return { return {
'source': SOURCE, 'external_id': doi, 'url': url, 'source': SOURCE, 'external_id': doi, 'url': url,
@@ -120,7 +130,7 @@ def publisher_row(chapter, toc, evidence):
'book_title': BOOK_TITLE, 'book_doi': toc['book_doi'], 'lncs_volume': toc['lncs_volume'], 'book_title': BOOK_TITLE, 'book_doi': toc['book_doi'], 'lncs_volume': toc['lncs_volume'],
'authors_citation_raw': chapter['authors_citation_raw'], 'first_page': chapter['first_page'], 'authors_citation_raw': chapter['authors_citation_raw'], 'first_page': chapter['first_page'],
'last_page': chapter['last_page'], 'publisher_chapter_evidence': evidence, 'last_page': chapter['last_page'], 'publisher_chapter_evidence': evidence,
'publisher_contents_evidence': {'url': toc['evidence_url'], 'sha256': toc['evidence_sha256']}, 'publisher_contents_evidence': contents_evidence,
'scope': 'Refereed conference proceedings, LNCS 17001–17078.', 'scope': 'Refereed conference proceedings, LNCS 17001–17078.',
'original_decision_signature_verified': False, 'original_decision_signature_verified': False,
}, },
+129
View File
@@ -0,0 +1,129 @@
"""Inventory KDD 2026 programme DOI, cycle and track observations."""
import argparse
from collections import Counter
from hashlib import sha256
from html.parser import HTMLParser
import json
from pathlib import Path
import re
SOURCE_URL = 'https://kdd2026.kdd.org/papers/'
CYCLES = {1: 'july', 2: 'february'}
TRACKS = {
'rtp': 'research', 'ads': 'applied_data_science', 'dtb': 'datasets_and_benchmarks',
'ais': 'ai_for_sciences', 'bsi': 'blue_sky_ideas',
}
DOI_URL = re.compile(r'https://doi\.org/(10\.1145/\d+\.\d+)')
WITHDRAWN = re.compile(
r'^\*\*WITHDRAWN/New SUBMISSION ID for Authorship Order Change (\d{1,2}/\d{1,2}/26) -- (.+)$'
)
OTHER_STATUS = re.compile(r'^[*\[(!#\s]*(?:withdrawn|rejected|desk[\s_-]+rejected|retracted|cancell?ed)\b', re.I)
class ProgrammeScripts(HTMLParser):
def __init__(self):
super().__init__()
self.current = None
self.scripts = []
def handle_starttag(self, tag, attrs):
if tag == 'script' and 'src' not in dict(attrs):
self.current = []
def handle_data(self, data):
if self.current is not None:
self.current.append(data)
def handle_endtag(self, tag):
if tag == 'script' and self.current is not None:
self.scripts.append(''.join(self.current))
self.current = None
def cycle_rows(scripts, cycle):
pattern = re.compile(r'\bconst\s+cycle' + str(cycle) + r'Papers\s*=\s*(\[)')
matches = [(code, match) for code in scripts for match in pattern.finditer(code)]
if len(matches) != 1:
raise ValueError(f'Expected one KDD programme array for cycle {cycle}')
code, match = matches[0]
body = code[match.start(1):]
rows, consumed = json.JSONDecoder().raw_decode(body)
if not rows or not body[consumed:].lstrip().startswith('.map((paper, index)'):
raise ValueError(f'Unexpected KDD programme array boundary for cycle {cycle}')
return rows
def inventory(raw):
parser = ProgrammeScripts()
parser.feed(raw.decode('utf-8'))
parser.close()
records = {}
source_counts = {}
for cycle, cycle_name in CYCLES.items():
rows = cycle_rows(parser.scripts, cycle)
source_counts[cycle_name] = Counter()
for row in rows:
if (not isinstance(row, dict) or row.keys() != {'track', 'title', 'url', 'authors'}
or not all(isinstance(value, str) for value in row.values())
or row['track'] not in TRACKS or not row['title'].strip()):
raise ValueError('Unexpected KDD programme row')
match = DOI_URL.fullmatch(row['url'])
if not match:
raise ValueError('Invalid DOI link in KDD programme')
doi = match[1]
track = TRACKS[row['track']]
source_counts[cycle_name][track] += 1
row_hash = sha256(json.dumps(row, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
if doi in records:
entry = records[doi]
if (entry['cycle'], entry['source_row_sha256']) != (cycle_name, row_hash):
raise ValueError('Conflicting duplicate DOI in KDD programme: ' + doi)
entry['occurrences'] += 1
continue
status = 'listed'
withdrawn = WITHDRAWN.fullmatch(row['title'])
if withdrawn:
status = 'withdrawn_with_replacement_mentioned'
elif OTHER_STATUS.match(row['title']):
raise ValueError('Unrecognized status annotation in KDD programme: ' + doi)
records[doi] = {
'doi': doi, 'doi_url': row['url'], 'source_title': row['title'],
'title': withdrawn[2] if withdrawn else row['title'],
'cycle': cycle_name, 'track': track, 'programme_status': status,
'authors_listed': bool(row['authors'].strip()), 'occurrences': 1,
'source_row_sha256': row_hash,
}
if withdrawn:
records[doi]['source_withdrawal_date_literal'] = withdrawn[1]
ordered = [records[doi] for doi in sorted(records)]
return ordered, source_counts
def collect(snapshot, expected_sha256, out):
snapshot = Path(snapshot)
with snapshot.open('rb') as stream:
raw = stream.read(4_000_001)
if len(raw) > 4_000_000 or sha256(raw).hexdigest() != expected_sha256:
raise ValueError('KDD programme snapshot hash or size differs')
rows, cycle_counts = inventory(raw)
result = {
'source': SOURCE_URL, 'snapshot_sha256': expected_sha256,
'source_rows_by_cycle_and_track': cycle_counts, 'unique_dois': len(rows),
'identical_repeat_rows': sum(row['occurrences'] - 1 for row in rows),
'programme_status_counts': Counter(row['programme_status'] for row in rows),
'records': rows, 'database_written': False,
}
with Path(out).open('x', encoding='utf-8') as stream:
json.dump(result, stream, ensure_ascii=False, indent=2)
stream.write('\n')
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--snapshot', required=True)
parser.add_argument('--snapshot-sha256', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
result = collect(args.snapshot, args.snapshot_sha256, args.out)
print(json.dumps({k: result[k] for k in ('unique_dois', 'identical_repeat_rows', 'programme_status_counts')}))
+11 -2
View File
@@ -103,6 +103,8 @@ def collect_volume(volume,cache,out,allowed=None):
path = Path(out)/f'pmlr-v{volume}.jsonl' path = Path(out)/f'pmlr-v{volume}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True) path.parent.mkdir(parents=True,exist_ok=True)
mixed_icml = venue == 'ICML' and year >= 2024 mixed_icml = venue == 'ICML' and year >= 2024
corl_blue_sky_volume = volume == 164 and venue == 'CORL' and year == 2021
corl_section_tracks = {'contributed papers': 'main', 'blue sky papers': 'blue_sky'}
mapping, track_program = collect_track_evidence(year, cache) if mixed_icml else ({}, None) mapping, track_program = collect_track_evidence(year, cache) if mixed_icml else ({}, None)
records = {} records = {}
prefaces = set() prefaces = set()
@@ -120,7 +122,8 @@ def collect_volume(volume,cache,out,allowed=None):
raise ValueError('Paper metadata has no publication ID: '+member_name) raise ValueError('Paper metadata has no publication ID: '+member_name)
external = f"v{volume}/{row['id']}" external = f"v{volume}/{row['id']}"
# PMLR also gives volume prefaces the inproceedings layout. # PMLR also gives volume prefaces the inproceedings layout.
if str(row.get('section') or '').strip().casefold() == 'preface': section = str(row.get('section') or '').strip()
if section.casefold() == 'preface':
prefaces.add(external) prefaces.add(external)
continue continue
ids = {} ids = {}
@@ -134,6 +137,12 @@ def collect_volume(volume,cache,out,allowed=None):
value = value.split('id=')[-1].split('&')[0] value = value.split('id=')[-1].split('&')[0]
ids[key] = value ids[key] = value
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}} record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
if corl_blue_sky_volume:
section_track = corl_section_tracks.get(section.casefold())
if section_track is None:
raise ValueError(f'Unrecognized CoRL 2021 section for {external}: {section!r}')
record['track'] = section_track
record['metadata']['pmlr_section'] = section
version={'path':member_name,'date':row.get('date')} version={'path':member_name,'date':row.get('date')}
if external in records: if external in records:
previous=records[external] previous=records[external]
@@ -163,7 +172,7 @@ def collect_volume(volume,cache,out,allowed=None):
with tempfile.TemporaryDirectory(prefix=f'.pmlr-v{volume}-', dir=path.parent) as temporary: with tempfile.TemporaryDirectory(prefix=f'.pmlr-v{volume}-', dir=path.parent) as temporary:
stage = Path(temporary) stage = Path(temporary)
for current_track, rows in sorted(grouped.items()): for current_track, rows in sorted(grouped.items()):
filename = f'pmlr-v{volume}-{current_track}.jsonl' if mixed_icml else path.name filename = f'pmlr-v{volume}-{current_track}.jsonl' if mixed_icml or current_track != track else path.name
output = stage / filename output = stage / filename
output.write_text(''.join(json.dumps(row, ensure_ascii=False, default=str)+'\n' for row in rows)) output.write_text(''.join(json.dumps(row, ensure_ascii=False, default=str)+'\n' for row in rows))
absent = sum(not (row.get('abstract') or '').strip() for row in rows) absent = sum(not (row.get('abstract') or '').strip() for row in rows)
+21 -1
View File
@@ -1,4 +1,6 @@
"""Versioned publication-source scopes and guards against stale source refreshes.""" """Versioned publication-source scopes and guards against stale source refreshes."""
import base64
import gzip
import json import json
import re import re
@@ -6,6 +8,24 @@ TABLE = 'source_scope_revisions'
MARKER = 'source_scope_revisions_v1' MARKER = 'source_scope_revisions_v1'
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url', COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
'parent_sha256', 'records_json', 'plan_json', 'applied_at') 'parent_sha256', 'records_json', 'plan_json', 'applied_at')
COMPRESSED_PREFIX = 'gzip+base64:v1:'
COMPRESSION_THRESHOLD = 1024 * 1024
def encode_payload(value):
"""Compress large revision payloads before sending them to the database."""
text = json.dumps(value, ensure_ascii=False, separators=(',', ':'))
raw = text.encode('utf-8')
if len(raw) < COMPRESSION_THRESHOLD:
return text
return COMPRESSED_PREFIX + base64.b64encode(gzip.compress(raw, compresslevel=1, mtime=0)).decode('ascii')
def decode_payload(text):
if text.startswith(COMPRESSED_PREFIX):
packed = base64.b64decode(text[len(COMPRESSED_PREFIX):], validate=True)
return json.loads(gzip.decompress(packed))
return json.loads(text)
def available(c): def available(c):
@@ -60,7 +80,7 @@ def current_rows(c, source_id, cache):
records = {}; scopes = {} records = {}; scopes = {}
for signature in signatures: for signature in signatures:
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0] raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
scopes[signature] = json.loads(raw) scopes[signature] = decode_payload(raw)
for row in scopes[signature]: for row in scopes[signature]:
ext = row['external_id'] ext = row['external_id']
if ext in records and records[ext] != row: if ext in records and records[ext] != row:
@@ -0,0 +1,222 @@
"""Attach reviewed ECCV proceedings evidence to an existing programme attempt."""
import argparse
from contextlib import closing
from hashlib import sha256
import json
from pathlib import Path
import re
from paper_atlas.audits.source_identity import entity_packet
from paper_atlas.collectors.eccv2026 import SOURCE as PROGRAMME
from paper_atlas.collectors.eccv2026_publications import SOURCE as PUBLISHER
from paper_atlas.db.backend import database_arguments, open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.submissions import identities_for_row, timestamp
from paper_atlas.ingestion.edition_tracks import curation
from paper_atlas.ingestion.records import source_record_count
MARKER = 'eccv_publication_reconciliation_v1:'
PUBLISHER_COUNT = 2835
def require(condition, message):
if not condition:
raise ValueError(message)
def packet_digest(packet):
return sha256(json.dumps(packet, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def validate_plan(plan):
require(plan.get('status') == 'readonly_same_submission_reconciliation_plan'
and plan.get('database_written') is False, 'A reviewed read-only plan is required')
rows = plan['plans']
require(0 < len(rows) <= 1000 and len(rows) == plan['guarded_same_submission_plans'],
'Unexpected approved plan size')
for key in ('doi', 'retained_paper_id', 'retained_submission_id', 'retained_programme_record_id'):
require(len({entry[key] for entry in rows}) == len(rows), 'Repeated reviewed ' + key)
for entry in rows:
doi = entry['doi']
row = entry['source_row']
url = 'https://link.springer.com/chapter/' + doi
evidence = row['metadata']['publisher_chapter_evidence']
require(re.fullmatch(r'10\.1007/978-3-032-\d{5}-[0-9Xx]_[1-9]\d*', doi)
and (row['source'], row['external_id'], row['url'], row['evidence_url']) ==
(PUBLISHER, doi, url, url)
and (row['venue'], row['year'], row['track'], row['status']) ==
('ECCV', 2026, 'main', 'accepted')
and row['identifiers'] == {'doi': doi}
and identities_for_row(row) == [('publication_doi', doi)]
and isinstance(row['abstract'], str) and row['abstract'].strip()
and re.fullmatch(r'[0-9a-f]{64}', evidence['sha256'])
and re.fullmatch(r'[0-9a-f]{64}', entry['before_packet_sha256']),
'Invalid reviewed publisher observation')
return rows
def existing_coverage(conn, edition, source):
rows = conn.execute('''SELECT list_url,status,expected_count,imported_count,cursor,last_success_at
FROM edition_sources WHERE edition_id=? AND source_id=?''', (edition, source)).fetchall()
require(len(rows) <= 1, 'Source coverage is ambiguous')
actual = source_record_count(conn, source, edition)
if not rows:
require(actual == 0, 'Linked records have no coverage listing')
return None
row = rows[0]
require(row[1] == 'partial' and row[3] == actual, 'Coverage differs from linked records')
return tuple(row)
def reconcile_attempt(store, entry, original, target, venue_id, event_key, venue_names, publisher, signature):
conn = store.conn
doi = entry['doi']
canonical_doi = doi.lower()
paper = entry['retained_paper_id']
submission = entry['retained_submission_id']
programme_record = entry['retained_programme_record_id']
row = entry['source_row']
before = entity_packet(conn, paper)
require(packet_digest(before) == entry['before_packet_sha256'], 'Reviewed paper history changed: ' + doi)
require(not before['source_records_truncated'] and not before['attempts_truncated']
and all(not attempt['decisions_truncated'] and not attempt['sources_truncated']
and all(not decision['sources_truncated'] for decision in attempt['decisions'])
for attempt in before['attempts']), 'Reviewed paper history exceeds verification limits')
same_event = [a for a in before['attempts'] if a['venue'].casefold() == 'eccv' and a['year'] == 2026]
require(len(same_event) == 1, 'Paper has multiple or missing ECCV 2026 attempts')
attempt = same_event[0]
require((attempt['id'], attempt['edition_id'], attempt['source_record_id']) ==
(submission, original, programme_record)
and [(d['id'], d['status']) for d in attempt['decisions']] ==
[(entry['retained_programme_decision_id'], 'accepted')]
and [(s['source_record_id'], s['source']) for s in attempt['sources']] ==
[(programme_record, PROGRAMME)], 'Original programme attempt differs')
programme = next(record for record in before['source_records'] if record['id'] == programme_record)
metadata = json.loads(programme['metadata_json'])
require(KEY not in metadata and attempt['source_submission_id'] == programme['external_id'],
'Programme observation already curated or ambiguous')
scope = curation({
'external_id': programme['external_id'], 'venue_names': venue_names, 'from_edition_id': original,
'year': 2026, 'from_track': 'program_unclassified', 'event_key': event_key,
'venue_id': venue_id, 'track': 'main', 'section': 'published ECCV refereed proceedings',
'evidence_url': row['evidence_url'], 'evidence_sha256': row['metadata']['publisher_chapter_evidence']['sha256'],
}, target, signature)
require(not conn.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=?', (submission,)).fetchone()
and not conn.execute("SELECT 1 FROM submission_identifiers WHERE edition_id=? AND scheme='publication_doi' AND value=?",
(target, doi)).fetchone()
and not conn.execute('SELECT 1 FROM source_records WHERE source_id=? AND external_id=?',
(publisher, doi)).fetchone(), 'Publication observation or submission identity already exists')
require(not conn.execute('SELECT 1 FROM identifiers WHERE scheme=? AND value=?',
(PUBLISHER, doi)).fetchone(), 'Publisher identity already exists')
owner = conn.execute("SELECT paper_id FROM identifiers WHERE scheme='doi' AND value=?", (canonical_doi,)).fetchone()
require(owner is None or owner[0] == paper, 'Publication DOI belongs to another paper')
if owner is None:
conn.execute("INSERT INTO identifiers(scheme,value,paper_id) VALUES('doi',?,?)", (canonical_doi, paper))
before['identifiers'].append({'scheme': 'doi', 'value': canonical_doi, 'paper_id': paper})
before['identifiers'].append({'scheme': PUBLISHER, 'value': doi, 'paper_id': paper})
before['identifiers'].sort(key=lambda identifier: (identifier['scheme'], identifier['value']))
require(conn.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',
(target, submission, original)).rowcount == 1, 'Programme attempt changed')
conn.execute("INSERT INTO submission_identifiers VALUES(?,'publication_doi',?,?)", (target, doi, submission))
metadata[KEY] = [scope]
programme['metadata_json'] = json.dumps(metadata, ensure_ascii=False)
conn.execute('UPDATE source_records SET metadata_json=? WHERE id=?',
(programme['metadata_json'], programme_record))
actual_paper, record = store.record(PUBLISHER, doi, row['url'], row['title'], row['abstract'],
row['authors'], row['identifiers'], row['metadata'])
require(actual_paper == paper, 'Unexpected canonical paper')
store.record_tags(paper, record, PUBLISHER, row['tags'])
actual_edition = store.decision(paper, record, 'ECCV', 2026, 'main', doi, row['evidence_url'],
'accepted', row['raw_decision'], identities=identities_for_row(row),
observed_at=row['observed_at'], event_key=event_key)
require(actual_edition == target, 'Unexpected target edition')
after = entity_packet(conn, paper)
published = [r for r in after['source_records'] if r['id'] == record]
require(len(published) == 1 and published[0]['external_id'] == doi
and published[0]['abstract'] == row['abstract'], 'Publication source is missing or changed')
after['source_records'].remove(published[0])
moved = next(a for a in after['attempts'] if a['id'] == submission)
attached = [s for s in moved['sources'] if s['source_record_id'] == record]
observed = [d for d in moved['decisions'] if any(s['source_record_id'] == record for s in d['sources'])]
require(len(attached) == len(observed) == 1 and attached[0]['source_submission_id'] == doi
and observed[0]['status'] == 'accepted' and observed[0]['evidence_url'] == row['evidence_url']
and len(observed[0]['sources']) == 1, 'Publisher evidence did not attach to the existing submission')
moved['sources'].remove(attached[0])
moved['decisions'].remove(observed[0])
attempt.update(edition_id=target, track='main')
require(before == after, 'Existing paper or submission history changed')
def apply(store, plan, signature, previous_count):
rows = validate_plan(plan)
conn = store.conn
marker = MARKER + signature
require(not conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (marker,)).fetchone(),
'Reconciliation already applied')
edition_rows = conn.execute('''SELECT e.track,e.id,e.venue_id,e.event_key
FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym='ECCV' AND e.year=2026
AND e.track IN ('main','program_unclassified')''').fetchall()
require(len(edition_rows) == 2 and {r[0] for r in edition_rows} == {'main', 'program_unclassified'},
'Expected two unambiguous reviewed editions')
editions = {r[0]: (r[1], r[2], r[3]) for r in edition_rows}
original, venue, event = editions['program_unclassified']
target, target_venue, target_event = editions['main']
require(venue == target_venue and event == target_event
and all((r['move_from_edition_id'], r['move_to_edition_id']) == (original, target) for r in rows),
'Reviewed edition identity differs')
names = ['ECCV', *(r[0] for r in conn.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (venue,)))]
sources = {r[0]: r[1] for r in conn.execute('SELECT name,id FROM sources WHERE name IN (?,?)', (PROGRAMME, PUBLISHER))}
programme, publisher = sources[PROGRAMME], sources[PUBLISHER]
old = existing_coverage(conn, original, programme)
main = existing_coverage(conn, target, programme)
published = existing_coverage(conn, target, publisher)
require(old is not None and old[3] >= len(rows) and (main is None or main[0] == old[0])
and published is not None and published[2:4] == (PUBLISHER_COUNT, previous_count)
and previous_count + len(rows) <= PUBLISHER_COUNT, 'Reviewed coverage parent differs')
for entry in rows:
reconcile_attempt(store, entry, original, target, venue, event, names, publisher, signature)
remaining = old[3] - len(rows)
imported = previous_count + len(rows)
main_count = (main[3] if main else 0) + len(rows)
for edition, source, expected in ((original, programme, remaining),
(target, programme, main_count), (target, publisher, imported)):
require(source_record_count(conn, source, edition) == expected, 'Reconciled coverage differs from linked records')
conn.execute('UPDATE edition_sources SET imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
(remaining, original, programme, old[0]))
conn.execute('''INSERT INTO edition_sources
(edition_id,source_id,list_url,status,imported_count,cursor,last_success_at) VALUES(?,?,?,'partial',?,?,?)
ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET imported_count=excluded.imported_count''',
(target, programme, old[0], main_count, old[4], old[5]))
conn.execute('''UPDATE edition_sources SET imported_count=?,last_success_at=?
WHERE edition_id=? AND source_id=? AND list_url=?''', (imported, timestamp(), target, publisher, published[0]))
conn.execute('INSERT INTO schema_migrations(name) VALUES(?)', (marker,))
return {'status': 'applied', 'plan_sha256': signature, 'programme_main': main_count,
'programme_unclassified': remaining, 'springer_main': imported, 'new_papers': 0, 'new_submissions': 0}
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
database_arguments(parser)
parser.add_argument('--plan', type=Path, required=True)
parser.add_argument('--plan-sha256', required=True)
parser.add_argument('--springer-previous-count', type=int, required=True)
args = parser.parse_args(argv)
raw = args.plan.read_bytes()
signature = sha256(raw).hexdigest()
require(signature == args.plan_sha256, 'Plan hash differs')
plan = json.loads(raw)
validate_plan(plan)
store = open_store(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema)
with closing(store.conn) as conn:
try:
if not getattr(conn, 'postgres', False):
conn.commit()
conn.execute('BEGIN IMMEDIATE')
result = apply(store, plan, signature, args.springer_previous_count)
conn.commit()
except BaseException:
conn.rollback()
raise
print(json.dumps(result), flush=True)
+8 -5
View File
@@ -203,7 +203,7 @@ def changes(c, inventory, before):
def proposal(c, inventory, identity): def proposal(c, inventory, identity):
before = capture(c, inventory) before = capture(c, inventory)
if before['parent_sha256']: if before['parent_sha256']:
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0]) saved = versions.decode_payload(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']): if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
raise ValueError('Previous snapshot differs from active source version') raise ValueError('Previous snapshot differs from active source version')
records, coverage = changes(c, inventory, before) records, coverage = changes(c, inventory, before)
@@ -241,8 +241,8 @@ def verify(c, plan, signature):
if after['coverage'] != expected_coverage: if after['coverage'] != expected_coverage:
raise ValueError('Reconciled coverage differs; roll back') raise ValueError('Reconciled coverage differs; roll back')
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone() stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
if (not stored or json.loads(stored[0]) != plan['inventory']['rows'] if (not stored or versions.decode_payload(stored[0]) != plan['inventory']['rows']
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']): or versions.decode_payload(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
raise ValueError('Versioned source or complete before-images changed') raise ValueError('Versioned source or complete before-images changed')
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']), return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])} 'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
@@ -269,6 +269,9 @@ def apply_plan(c, plan):
signature = digest(plan) signature = digest(plan)
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone(): if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)} return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
inventory = plan['inventory']
records_payload = versions.encode_payload(inventory['rows'])
plan_payload = versions.encode_payload(plan)
if getattr(c, 'postgres', False): if getattr(c, 'postgres', False):
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE') c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
if proposal(c, plan['inventory'], plan['database']) != plan: if proposal(c, plan['inventory'], plan['database']) != plan:
@@ -284,10 +287,10 @@ def apply_plan(c, plan):
before = row['before'] before = row['before']
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?', c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
(row['expected_count'], row['imported_count'], before[0], before[1], before[2])) (row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
inventory = plan['inventory']
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json) c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'], VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False))) inventory['list_url'], plan['before']['parent_sha256'],
records_payload, plan_payload))
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)} return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
+31
View File
@@ -54,6 +54,27 @@ def read_page(index, url):
return raw, evidence return raw, evidence
def require_display_authors(nodes, authors):
matching_article_lists = []
for node in nodes:
if node.tag != 'script' or node.attrs.get('type') != 'application/ld+json':
continue
try:
article = json.loads(''.join(node.children))
except json.JSONDecodeError:
continue
if not isinstance(article, dict) or article.get('@type') != 'ScholarlyArticle':
continue
entries = article.get('author')
if not isinstance(entries, list) or not all(
isinstance(entry, dict) and isinstance(entry.get('name'), str) for entry in entries
):
raise ValueError('Unreviewed Springer article author list')
matching_article_lists.append([clean(entry['name']) for entry in entries])
if matching_article_lists != [authors]:
raise ValueError('Single-name author requires matching Springer display author list')
def parse_chapter(html, doi, *, book_title, publication_year): def parse_chapter(html, doi, *, book_title, publication_year):
nodes = list(Document(html).root.walk()) nodes = list(Document(html).root.walk())
meta = defaultdict(list) meta = defaultdict(list)
@@ -73,12 +94,22 @@ def parse_chapter(html, doi, *, book_title, publication_year):
raise ValueError('Missing named chapter authors') raise ValueError('Missing named chapter authors')
authors = [] authors = []
for raw in raw_authors: for raw in raw_authors:
if ',' not in raw:
name = clean(raw)
if len(name.split()) != 1 or not any(character.isalpha() for character in name):
raise ValueError('Unreviewed citation author format')
authors.append(name)
continue
if raw.count(',') != 1: if raw.count(',') != 1:
raise ValueError('Unreviewed citation author format') raise ValueError('Unreviewed citation author format')
family, given = [clean(s) for s in raw.split(',', 1)] family, given = [clean(s) for s in raw.split(',', 1)]
if not family or not given: if not family or not given:
raise ValueError('Incomplete citation author') raise ValueError('Incomplete citation author')
authors.append({'given':given, 'family':family}) authors.append({'given':given, 'family':family})
if any(isinstance(author, str) for author in authors):
display_names = [author if isinstance(author, str) else author['given']+' '+author['family']
for author in authors]
require_display_authors(nodes, display_names)
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content'] abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
if len(abstract_nodes) > 1: if len(abstract_nodes) > 1:
raise ValueError('Ambiguous abstract section') raise ValueError('Ambiguous abstract section')
+4
View File
@@ -26,6 +26,10 @@ class Result:
def rowcount(self): def rowcount(self):
return self.cursor.rowcount return self.cursor.rowcount
@property
def description(self):
return self.cursor.description
class WriteConnection: class WriteConnection:
postgres = True postgres = True
+14 -1
View File
@@ -6,7 +6,7 @@ import sqlite3
import tempfile import tempfile
import unittest import unittest
from paper_atlas.collectors.eccv2026_publications import BOOK_TITLE, SOURCE, collect from paper_atlas.collectors.eccv2026_publications import BOOK_TITLE, SOURCE, collect, publisher_row
from paper_atlas.db.store import Store from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file from paper_atlas.ingestion.records import ingest_file
@@ -114,6 +114,19 @@ class ECCV2026PublicationsTests(unittest.TestCase):
with self.assertRaises(FileExistsError): with self.assertRaises(FileExistsError):
collect(root / 'out', **source) collect(root / 'out', **source)
def test_numeric_superscript_wrappers_preserve_both_title_literals(self):
chapter = {'doi': FIRST_DOI, 'title': 'Example $$^2$$ : Formal title',
'authors': [], 'abstract': 'Published abstract.', 'authors_citation_raw': [],
'first_page': '1', 'last_page': '10'}
toc = {'title': r'Example \(^2\) : Formal title', 'book_doi': FIRST_BOOK, 'lncs_volume': 17001,
'evidence_url': 'https://link.springer.com/book/' + FIRST_BOOK, 'evidence_sha256': 'a' * 64}
evidence = {'observed_at': '2026-11-18T00:00:00Z'}
row = publisher_row(chapter, toc, evidence)
self.assertEqual(row['title'], chapter['title'])
self.assertEqual(row['metadata']['publisher_contents_evidence']['title_literal'], toc['title'])
with self.assertRaisesRegex(ValueError, 'title differs'):
publisher_row(chapter, {**toc, 'title': r'Example \(^3\) : Formal title'}, evidence)
def test_changed_contents_receipt_stops_before_output(self): def test_changed_contents_receipt_stops_before_output(self):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp) root = Path(tmp)
+68
View File
@@ -0,0 +1,68 @@
import io
import json
import sqlite3
import unittest
from unittest.mock import Mock, patch
from paper_atlas.audits import global_identity
class GlobalIdentityCandidatesTest(unittest.TestCase):
def test_exact_same_title_and_near_title_require_author_evidence(self):
output = io.StringIO()
with sqlite3.connect(':memory:') as scratch:
scratch.execute('CREATE TABLE titles (id INTEGER, title_key TEXT, authors BLOB)')
scratch.executemany('INSERT INTO titles VALUES (?,?,?)', [
(1, 'astronomicaltransientcatalogue', b'first'),
(2, 'astronomicaltransientcatalogue', b'first'),
(3, 'astronomicaltransientcatalogues', b'first'),
(4, 'astronomicaltransientcatalogue', b'second'),
(5, 'astronomicaltransientcatalogue', None),
])
counts = global_identity._candidates(scratch, output)
records = [json.loads(row) for row in output.getvalue().splitlines()]
self.assertEqual(counts['exact_title_groups'], 1)
self.assertEqual(counts['exact_title_and_roster_papers'], 2)
self.assertEqual(counts['near_title_and_roster_pairs'], 2)
self.assertEqual(records[0]['paper_ids'], [1, 2])
self.assertEqual({tuple(r['paper_ids']) for r in records[1:]}, {(1, 3), (2, 3)})
def test_author_roster_limit_is_visible_in_receipt(self):
output = io.StringIO()
with sqlite3.connect(':memory:') as scratch:
scratch.execute('CREATE TABLE titles (id INTEGER, title_key TEXT, authors BLOB)')
scratch.executemany('INSERT INTO titles VALUES (?,?,?)', [
(1, 'astronomicaltransientcatalogue', b'first'),
(2, 'astronomicaltransientcatalogues', b'first'),
(3, 'astronomicaltransientcatalogued', b'first'),
])
with patch.object(global_identity, 'ROSTER_GROUP_LIMIT', 2):
counts = global_identity._candidates(scratch, output)
self.assertEqual(counts['oversized_rosters'], 1)
self.assertEqual(counts['papers_in_oversized_rosters'], 3)
self.assertEqual(counts['near_title_and_roster_pairs'], 0)
self.assertEqual(json.loads(output.getvalue())['kind'], 'oversized_roster')
def test_fingerprint_normalizes_supported_author_encodings(self):
strings = json.dumps(['Alice Doe', 'Chloë Li'])
records = json.dumps([['Li', 'Chloë'], {'given':'Alice','family':'Doe'}])
self.assertEqual(global_identity._author_fingerprint(strings),
global_identity._author_fingerprint(records))
self.assertIsNone(global_identity._author_fingerprint('[]'))
def test_doi_pattern_is_bound_as_a_parameter(self):
connection = Mock()
def execute(query, params):
self.assertEqual(query.count('?'), len(params))
if params:
pattern, = params
self.assertIn('https?', pattern)
result = Mock()
result.fetchone.return_value = (0,)
return result
connection.execute.side_effect = execute
self.assertEqual(global_identity._checks(connection)['normalized_doi_collisions'], 0)
+68
View File
@@ -0,0 +1,68 @@
from hashlib import sha256
import json
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
from paper_atlas.cli import main
from paper_atlas.collectors.kdd2026 import inventory
def row(number, track, title='Paper title', authors='Name (Affiliation)'):
return {'track': track, 'title': title, 'url': f'https://doi.org/10.1145/3770855.{number}', 'authors': authors}
def snapshot(july, february):
arrays = [
f'<script>const cycle{n}Papers = {json.dumps(rows)}.map((paper, index) => paper);</script>'
for n, rows in ((1, july), (2, february))
]
return '\n'.join(arrays).encode()
class KDD2026Tests(unittest.TestCase):
def test_cli_preserves_withdrawal_marker_and_identical_occurrences(self):
retained = row(1, 'rtp')
withdrawn = row(2, 'ads', '**WITHDRAWN/New SUBMISSION ID for Authorship Order Change 6/24/26 -- A revised work')
incomplete = row(3, 'ais', authors='')
raw = snapshot([retained, retained], [withdrawn, incomplete])
with TemporaryDirectory() as directory:
source = Path(directory) / 'snapshot.html'
source.write_bytes(raw)
out = source.with_suffix('.json')
args = ['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
sha256(raw).hexdigest(), '--out', str(out)]
main(args)
report = json.loads(out.read_bytes())
with self.assertRaises(FileExistsError):
main(args)
self.assertEqual(report['source_rows_by_cycle_and_track'], {
'july': {'research': 2}, 'february': {'applied_data_science': 1, 'ai_for_sciences': 1},
})
self.assertEqual(report['unique_dois'], 3)
self.assertEqual(report['identical_repeat_rows'], 1)
kept, replaced, sparse = report['records']
self.assertEqual(kept['occurrences'], 2)
self.assertEqual(replaced['programme_status'], 'withdrawn_with_replacement_mentioned')
self.assertEqual(replaced['title'], 'A revised work')
self.assertEqual(replaced['source_title'], withdrawn['title'])
self.assertEqual(replaced['source_withdrawal_date_literal'], '6/24/26')
self.assertNotIn('submission_id', replaced)
self.assertFalse(sparse['authors_listed'])
self.assertFalse(report['database_written'])
def test_conflicting_duplicate_and_unknown_status_fail(self):
with self.assertRaisesRegex(ValueError, 'Conflicting duplicate DOI'):
inventory(snapshot([row(1, 'rtp')], [row(1, 'ads')]))
with self.assertRaisesRegex(ValueError, 'Unrecognized status'):
inventory(snapshot([row(1, 'rtp')], [row(2, 'ads', '***WITHDRAWN -- Unknown format')]))
def test_hash_mismatch_leaves_no_output(self):
with TemporaryDirectory() as directory:
source = Path(directory) / 'snapshot.html'
source.write_bytes(snapshot([row(1, 'rtp')], [row(2, 'ads')]))
out = source.with_suffix('.json')
with self.assertRaisesRegex(ValueError, 'snapshot hash or size'):
main(['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
'0' * 64, '--out', str(out)])
self.assertFalse(out.exists())
+18
View File
@@ -60,6 +60,24 @@ class MetadataTests(unittest.TestCase):
self.assertEqual(result['files'][0]['count'], 1) self.assertEqual(result['files'][0]['count'], 1)
self.assertEqual([row['external_id'] for row in rows], ['v244/paper1']) self.assertEqual([row['external_id'] for row in rows], ['v244/paper1'])
def test_corl_2021_sections_keep_blue_sky_apart_from_contributed_papers(self):
config = b'shortname: CoRL\nyear: 2021\nbooktitle: Proceedings of the 5th Conference on Robot Learning\n'
paper = '---\nid: {id}\nlayout: inproceedings\ntitle: Paper\nabstract: Abstract\nsection: {section}\n---\n'
posts = [('one.md', paper.format(id='one', section='Contributed Papers')),
('two.md', paper.format(id='two', section='Blue Sky Papers'))]
with tempfile.TemporaryDirectory() as tmp, patch('paper_atlas.collectors.pmlr.fetch', return_value=config), patch(
'paper_atlas.collectors.pmlr.metadata_texts', return_value=iter(posts),
):
result = collect_volume(164, tmp, tmp)
self.assertEqual(result['track'], 'mixed')
files = {manifest['track']: Path(manifest['file']) for manifest in result['files']}
self.assertEqual({key: path.name for key, path in files.items()},
{'main': 'pmlr-v164.jsonl', 'blue_sky': 'pmlr-v164-blue_sky.jsonl'})
rows = {track: json.loads(path.read_text()) for track, path in files.items()}
self.assertEqual([(rows[t]['external_id'], rows[t]['track'], rows[t]['metadata']['pmlr_section'])
for t in ('main', 'blue_sky')],
[('v164/one', 'main', 'Contributed Papers'), ('v164/two', 'blue_sky', 'Blue Sky Papers')])
def test_conflicting_duplicate_posts_do_not_replace_previous_collection(self): def test_conflicting_duplicate_posts_do_not_replace_previous_collection(self):
config=b'shortname: CoRL\nyear: 2023\nbooktitle: CoRL\n' config=b'shortname: CoRL\nyear: 2023\nbooktitle: CoRL\n'
paper='---\nid: paper1\nlayout: inproceedings\ntitle: {title}\nabstract: Abstract\n---\n' paper='---\nid: paper1\nlayout: inproceedings\ntitle: {title}\nabstract: Abstract\n---\n'
+140
View File
@@ -0,0 +1,140 @@
from contextlib import closing, redirect_stdout
from hashlib import sha256
import io
import json
import os
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
import uuid
from paper_atlas.audits.source_identity import entity_packet
from paper_atlas.cli import main
from paper_atlas.db.edition_scopes import resolve
from paper_atlas.db.store import Store
from paper_atlas.ingestion.publication_reconciliation import PROGRAMME, PUBLISHER, packet_digest
from paper_atlas.postgres.client import ReadConnection, connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
class PublicationReconciliationTests(unittest.TestCase):
def setUp(self):
directory = TemporaryDirectory()
self.addCleanup(directory.cleanup)
self.root = Path(directory.name)
self.db = self.root / 'db.sqlite'
self.store = Store(self.db)
self.conn = self.store.conn
self.addCleanup(self.conn.close)
self.programme_url = 'https://eccv.ecva.net/Conferences/2026/AcceptedPapers'
self.publisher_url = 'https://link.springer.com/conference/eccv'
programme = self.store.source(PROGRAMME, self.programme_url)
publisher = self.store.source(PUBLISHER, self.publisher_url)
self.entries = [self.programme_attempt(str(i)) for i in range(1, 4)]
title = 'A distinct chapter already integrated in the main track'
paper, record = self.store.record(PUBLISHER, 'prior', self.publisher_url, title, 'Text', [])
self.target = self.store.accepted(paper, record, 'ECCV', 2026, 'main', 'prior', self.publisher_url)
self.original = self.entries[0]['move_from_edition_id']
for entry in self.entries:
entry['move_to_edition_id'] = self.target
self.conn.executemany('''INSERT INTO edition_sources
(edition_id,source_id,list_url,status,expected_count,imported_count) VALUES(?,?,?,'partial',?,?)''',
[(self.original, programme, self.programme_url, 2834, 3),
(self.target, publisher, self.publisher_url, 2835, 1)])
self.conn.commit()
def programme_attempt(self, number):
title = 'A programme observation for continuing research ' + number
url = self.programme_url + '/' + number
authors = ['Ada Example', 'Sam Example']
paper, record = self.store.record(PROGRAMME, number, url, title, 'Programme text', authors)
original = self.store.accepted(paper, record, 'ECCV', 2026, 'program_unclassified', number, url)
other, old_record = self.store.record('openreview', 'test-' + number, url, title, 'Programme text', authors)
self.assertEqual(other, paper)
self.store.decision(paper, old_record, 'ICLR', 2025, 'main', 'test-' + number, url, 'rejected', 'Reject')
packet = entity_packet(self.conn, paper)
attempt = next(row for row in packet['attempts'] if row['source_record_id'] == record)
doi = '10.1007/978-3-032-00000-1_' + number
chapter = 'https://link.springer.com/chapter/' + doi
row = {'source': PUBLISHER, 'external_id': doi, 'url': chapter, 'evidence_url': chapter,
'venue': 'ECCV', 'year': 2026, 'track': 'main', 'status': 'accepted',
'title': title, 'authors': authors, 'abstract': 'Publisher text',
'observed_at': '2026-11-18T00:00:00Z', 'raw_decision': 'Publication observation',
'identifiers': {'doi': doi},
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
'metadata': {'publisher_chapter_evidence': {'sha256': 'a' * 64}},
'tags': [{'namespace': PUBLISHER + ':format', 'value': 'chapter'}]}
return {'doi': doi, 'retained_paper_id': paper, 'retained_programme_record_id': record,
'retained_submission_id': attempt['id'], 'retained_programme_decision_id': attempt['decisions'][0]['id'],
'move_from_edition_id': original, 'source_row': row, 'before_packet_sha256': packet_digest(packet)}
def run_plan(self, entries, *database):
path = self.root / 'plan.json'
raw = json.dumps({'status': 'readonly_same_submission_reconciliation_plan', 'database_written': False,
'plans': entries, 'guarded_same_submission_plans': len(entries)}).encode()
path.write_bytes(raw)
main(['maintain', 'eccv-publication-reconciliation', *(database or ('--db', str(self.db))),
'--plan', str(path), '--plan-sha256', sha256(raw).hexdigest(), '--springer-previous-count', '1'])
def test_preserves_history_and_updates_all_three_coverage_scopes(self):
entry = self.entries[0]
before = entity_packet(self.conn, entry['retained_paper_id'])
self.run_plan([entry])
packet = entity_packet(self.conn, entry['retained_paper_id'])
self.assertEqual(packet['paper'], before['paper'])
programme, other = packet['attempts']
self.assertEqual((programme['id'], programme['edition_id']), (entry['retained_submission_id'], self.target))
self.assertEqual(other, before['attempts'][1])
self.assertEqual(programme['decisions'][0], before['attempts'][0]['decisions'][0])
self.assertEqual([row['status'] for row in programme['decisions']], ['accepted', 'accepted'])
self.assertEqual({r['source'] for r in programme['sources']}, {PROGRAMME, PUBLISHER})
self.assertEqual(len(packet['source_records']), 3)
self.assertEqual(resolve(self.conn, entry['retained_programme_record_id'], '1',
'ECCV', 2026, 'program_unclassified', None), self.target)
coverage = {(edition, source): (status, expected, imported)
for edition, source, status, expected, imported in self.conn.execute('''SELECT
e.track,s.name,c.status,c.expected_count,c.imported_count FROM edition_sources c
JOIN editions e ON e.id=c.edition_id JOIN sources s ON s.id=c.source_id''')}
self.assertEqual(coverage, {('program_unclassified', PROGRAMME): ('partial', 2834, 2),
('main', PROGRAMME): ('partial', None, 1),
('main', PUBLISHER): ('partial', 2835, 2)})
with self.assertRaisesRegex(ValueError, 'already applied'):
self.run_plan([entry])
def test_changed_history_rolls_back_earlier_observation(self):
first, second = self.entries[:2]
stale = {**second, 'before_packet_sha256': 'b' * 64}
packet = entity_packet(self.conn, first['retained_paper_id'])
with self.assertRaisesRegex(ValueError, 'Reviewed paper history changed'):
self.run_plan([first, stale])
self.assertEqual(entity_packet(self.conn, first['retained_paper_id']), packet)
self.assertIsNone(self.conn.execute(
"SELECT 1 FROM identifiers WHERE scheme='doi' AND value=?", (first['doi'],)).fetchone())
self.assertEqual(self.conn.execute('SELECT count(*) FROM edition_sources').fetchone()[0], 2)
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the remote PostgreSQL test service')
def test_postgres_write_verifies_history_from_the_read_backend(self):
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
schema = 'pa_eccv_' + uuid.uuid4().hex[:12]
def cleanup():
with connect(service) as conn:
for name, in conn.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
conn.execute('DROP SCHEMA ' + quote(name) + ' CASCADE')
self.addCleanup(cleanup)
self.conn.close()
with redirect_stdout(io.StringIO()):
migrate(self.db, service, self.root / 'migration.json', schema=schema, min_free_gib=0)
entry = self.entries[0]
with closing(ReadConnection(service, schema)) as conn:
before = entity_packet(conn, entry['retained_paper_id'])
reviewed = {**entry, 'before_packet_sha256': packet_digest(before)}
self.run_plan([reviewed], '--pg-service', service, '--pg-schema', schema)
with closing(ReadConnection(service, schema)) as conn:
packet = entity_packet(conn, entry['retained_paper_id'])
self.assertEqual(packet['attempts'][1], before['attempts'][1])
self.assertEqual(packet['attempts'][0]['id'], entry['retained_submission_id'])
self.assertEqual(packet['attempts'][0]['edition_id'], self.target)
self.assertEqual(len(packet['attempts'][0]['decisions']), 2)
+23 -1
View File
@@ -13,7 +13,7 @@ import uuid
from paper_atlas.db.backend import open_store from paper_atlas.db.backend import open_store
from paper_atlas.db.edition_scopes import KEY from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.source_revisions import available, require_row, require_manifest from paper_atlas.db.source_revisions import available, decode_payload, require_row, require_manifest
from paper_atlas.db.store import Store from paper_atlas.db.store import Store
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
from paper_atlas.ingestion.event_identity import database_identity, digest, values from paper_atlas.ingestion.event_identity import database_identity, digest, values
@@ -69,6 +69,28 @@ class RevisionCases:
def plan(self, c): def plan(self, c):
return proposal(c, self.inventory, database_identity(c, self.target.get('db'))) return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
def test_large_payload_round_trips_and_protects_next_revision(self):
for key in ('previous_rows', 'rows'):
self.inventory[key][-1]['metadata']['evidence'] = '来源原文' * 100000
with closing(self.store().conn) as c:
plan = self.plan(c)
applied = apply_plan(c, plan)
c.commit()
saved = c.execute('SELECT records_json,plan_json FROM source_scope_revisions').fetchone()
expected = (self.inventory['rows'], plan)
for i, value in enumerate(expected):
self.assertTrue(saved[i].startswith('gzip+base64:v1:'))
self.assertLess(len(saved[i]), len(json.dumps(value, ensure_ascii=False).encode()) // 10)
self.assertEqual(decode_payload(saved[i]), value)
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
require_row(c, sid, self.inventory['rows'][-1], {'list_url': URL}, {})
next_inventory = copy.deepcopy(self.inventory)
next_inventory['previous_rows'] = copy.deepcopy(self.inventory['rows'])
next_inventory['rows'][-1]['metadata']['reviewed'] = True
next_plan = proposal(c, next_inventory, database_identity(c, self.target.get('db')))
self.assertEqual(next_plan['before']['parent_sha256'], applied['version_sha256'])
def test_move_to_populated_target_preserves_full_history_and_before_images(self): def test_move_to_populated_target_preserves_full_history_and_before_images(self):
with closing(self.store().conn) as c: with closing(self.store().conn) as c:
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1) plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
+23 -1
View File
@@ -9,7 +9,7 @@ import tempfile
import unittest import unittest
from paper_atlas.collectors.springer_selected import ( from paper_atlas.collectors.springer_selected import (
BOOK, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index, BOOK, SCOPE, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
) )
from paper_atlas.db.store import Store from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file from paper_atlas.ingestion.records import ingest_file
@@ -153,6 +153,28 @@ class SpringerSelectedTests(unittest.TestCase):
html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"') html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"')
self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract']) self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract'])
def test_single_name_preserves_literal_and_requires_ordered_display_confirmation(self):
html = f"""
<meta name="citation_doi" content="{BOOK}_1">
<meta name="citation_inbook_title" content="{SCOPE['book_title']}">
<meta name="citation_publication_date" content="{SCOPE['publication_year']}">
<meta name="citation_title" content="Example">
<meta name="citation_author" content="Rajkumar">
<meta name="citation_author" content="Doe, Jane">
<meta name="citation_firstpage" content="1">
<meta name="citation_lastpage" content="5">
"""
article = {'@type':'ScholarlyArticle', 'author':[{'name':'Rajkumar'}, {'name':'Jane Doe'}]}
script = '<script type="application/ld+json">{}</script>'
chapter = parse_chapter(html+script.format(json.dumps(article)), BOOK+'_1')
self.assertEqual(chapter['authors'], ['Rajkumar', {'family':'Doe', 'given':'Jane'}])
self.assertEqual(chapter['authors_citation_raw'], ['Rajkumar','Doe, Jane'])
with self.assertRaisesRegex(ValueError, 'matching Springer display author list'):
parse_chapter(html, BOOK+'_1')
article['author'].reverse()
with self.assertRaisesRegex(ValueError, 'matching Springer display author list'):
parse_chapter(html+script.format(json.dumps(article)), BOOK+'_1')
def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self): def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self):
html = self.fixture['toc']['springer-toc1.html'] html = self.fixture['toc']['springer-toc1.html']
self.assertEqual(len(parse_toc(html)), 17) self.assertEqual(len(parse_toc(html)), 17)