Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dc4f61d2f6 | |||
| 04b2ed9e95 | |||
| 8c6a328a4d | |||
| 710d84a8bf | |||
| d138981da4 | |||
| ef8c1cfeb9 | |||
| 5e5d07d086 | |||
| 50ffc2d4b6 | |||
| e454ed51ec | |||
| 32a53977ee | |||
| da5b10f2a6 | |||
| 6e5202c81a | |||
| ae8bcd4c14 | |||
| 3acb738aa8 | |||
| cbf7bab8ca | |||
| 4d97822d9e | |||
| bc01f90e70 |
@@ -47,6 +47,20 @@ jobs:
|
||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||
outside = root / 'outside-checkout'
|
||||
outside.mkdir()
|
||||
kdd_snapshot = pathlib.Path('/data/paper-index/validation/kdd2026-official-papers-html-v1.html')
|
||||
if kdd_snapshot.is_file():
|
||||
kdd_output = root / 'installed-kdd2026.json'
|
||||
subprocess.run([python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'kdd2026-program',
|
||||
'--snapshot', str(kdd_snapshot), '--snapshot-sha256',
|
||||
'47ee82841dedf034614348d7fdb29714e8ad62fca8c6811e1d856845588f145c',
|
||||
'--out', str(kdd_output)], cwd=outside, check=True)
|
||||
kdd = json.loads(kdd_output.read_bytes())
|
||||
assert kdd['unique_dois'] == 1408 and kdd['identical_repeat_rows'] == 7
|
||||
assert {key:sum(counts.values()) for key,counts in kdd['source_rows_by_cycle_and_track'].items()} == {
|
||||
'july':256, 'february':1159}
|
||||
held, = [entry for entry in kdd['records'] if entry['programme_status'] != 'listed']
|
||||
assert held['doi'] == '10.1145/3770855.3818478' and held['source_withdrawal_date_literal'] == '6/24/26'
|
||||
print('KDD 2026 official cycle and withdrawal fixture passed', flush=True)
|
||||
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
|
||||
if cikm_snapshots.is_dir():
|
||||
cikm_output = root / 'installed-cikm2026'
|
||||
|
||||
@@ -147,8 +147,10 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
||||
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
[KDD 2026 官方目录](docs/kdd2026-program.md)可以按 DOI 核对两个投稿周期、分轨与目录中的撤回更换标注。
|
||||
|
||||
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||
|
||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||
|
||||
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。
|
||||
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。同届节目投稿可按独立审核计划附加出版证据,保留原投稿 ID 和既往决定。
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,35 @@
|
||||
# CLeaR 2022–2026 逐篇覆盖核验
|
||||
|
||||
2026 年 11 月 20 日重新读取五卷 PMLR 目录,目录哈希与此前保存的出版目录核查一致。逐个出版 ID 对照 PostgreSQL。243 条全部关联到对应会年的主会录用。每个出版 ID 有一条录用关联,243 条来源记录均有摘要。 [逐条回执](clear-2022-2026-record-coverage.json)
|
||||
|
||||
| 年份 | 目录 | 录用关联 |
|
||||
| --- | --- | --- |
|
||||
| 2022 | [PMLR 177](https://proceedings.mlr.press/v177/) | 48/48 |
|
||||
| 2023 | [PMLR 213](https://proceedings.mlr.press/v213/) | 39/39 |
|
||||
| 2024 | [PMLR 236](https://proceedings.mlr.press/v236/) | 45/45 |
|
||||
| 2025 | [PMLR 275](https://proceedings.mlr.press/v275/) | 58/58 |
|
||||
| 2026 | [PMLR 323](https://proceedings.mlr.press/v323/) | 53/53 |
|
||||
|
||||
2026 年投稿规则允许录用作者用一页扩展摘要代替全文发表。核验覆盖上述正式出版目录。节目单上可能另有展示记录,需按主办方的逐篇证据单独核对。 [2026 官方征稿](https://www.cclear.cc/2026/CallforPapers)
|
||||
|
||||
五届公开 OpenReview 分组均可读取。2022–2023 分组未提供本次查询所需的拒稿赛道字段。2024–2026 分组将撤回、初筛拒稿默认公开标记设为 false,并提供拒稿赛道标识。对 2024 拒稿赛道的第一条匿名请求返回 HTTP 403,随后停止。2025–2026 的具体公开拒稿可见性因此仍待核。五届当前库内无逐篇负面决定。本次为只读核验。 [OpenReview 会议目录](https://openreview.net/venue?id=cclear.cc%2FCLeaR) · [逐年核验回执](clear-2022-2026-record-coverage.json)
|
||||
|
||||
## 2023 数据集赛道
|
||||
|
||||
主办方另外举行了因果基准数据集赛道,以邮件收件,并公开了 6 条录用。六条均已入库并通过常规查询回查。它们单列在 `dataset` 赛道,原主会 39 条录用关联通过写入前后复验。 [官方征集](https://www.cclear.cc/2023/CallforDatasets) · [官方录用名单](https://www.cclear.cc/2023/AcceptedDatasets) · [逐条查询回执](clear-2023-dataset-roster.json)
|
||||
|
||||
SCARY 关联到同题同作者的 arXiv 作品。SpaCE 的共同作者明确将 CLeaR 2023 列为早期版本,已关联到同一作品并保留 ICLR 2024 的录用历史和两个版本的署名。另 4 篇按官方名单暂定建档,近似题名查询超时,仍需最终去重复核。六条名单来源记录的摘要留空。名单仅公布录用,本次未新增负面决定。 [SCARY 预印本](https://arxiv.org/abs/2304.14109) · [SpaCE 共同作者说明](https://anatrisovic.com/docs/atrisovic-cv.pdf)
|
||||
|
||||
## 主会录用名单和日程
|
||||
|
||||
以主办方逐篇名单和日程补挂展示来源。原投稿及决定继续使用原 ID。展示形式以来源标签保存,同篇可同时有口头报告和海报。 [逐篇回执](clear-2022-2026-main-programme.json)
|
||||
|
||||
| 年份 | 已关联作品 | 来源及展示情况 |
|
||||
| --- | ---: | --- |
|
||||
| 2022 | 49 | 9 条口头报告、40 条海报。另于 PMLR 48 条之外补入一条扩展摘要录用。 |
|
||||
| 2023 | 39 | 8 条口头报告、31 条海报。一个节目题名变体由作者技术报告核对。 |
|
||||
| 2024 | 45 | 59 项日程展示,含同篇的多项展示。5 条日程题名变体按正式出版页和作者资料核对。 |
|
||||
| 2025 | 58 | 9 条口头报告和 58 条海报展示,同篇共享投稿。 |
|
||||
| 2026 | 9 | 9 条日程明列题名的口头报告。海报只列投稿编号,逐篇对应仍待核。 |
|
||||
|
||||
2024 与 2025 的网站录用页指向 2023 名单,使用各年完整版日程。2026 数据集赛道征稿规定筛选后在独立网站发布并开放投票,逐篇结果名单仍待定位。 [2024 日程](https://www.cclear.cc/2024/FullAgenda) · [2025 日程](https://www.cclear.cc/2025/FullAgenda) · [2026 日程](https://www.cclear.cc/2026/FullAgenda) · [2026 数据集征集](https://www.cclear.cc/2026/CallforDatasets)
|
||||
@@ -0,0 +1,178 @@
|
||||
{
|
||||
"status": "committed_and_readonly_verified",
|
||||
"snapshot_at": "2026-11-20 13:14:53.43621+00",
|
||||
"roster_status": "complete",
|
||||
"expected": 6,
|
||||
"imported": 6,
|
||||
"new": 5,
|
||||
"records": [
|
||||
{
|
||||
"external_id": "bizeul23a",
|
||||
"title": "3DIdentBox: A Toolbox for Identifiability Benchmarking",
|
||||
"authors_json": "[\"Alice Bizeul\", \"Imant Daunhawer\", \"Emanuele Palumbo\", \"Bernhard Schölkopf\", \"Alexander Marx\", \"Julia E. Vogt\"]",
|
||||
"paper_id": 3572631,
|
||||
"source_record_id": 3824788,
|
||||
"submission_id": 541329,
|
||||
"decision_id": 580918,
|
||||
"status": "accepted",
|
||||
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
|
||||
"decision_date": null
|
||||
},
|
||||
{
|
||||
"external_id": "chen23a",
|
||||
"title": "The Structurally Complex with Additive Parent Causality (SCARY) Dataset",
|
||||
"authors_json": "[\"Jarry Chen\", \"Haytham M. Fayek\"]",
|
||||
"paper_id": 1834131,
|
||||
"source_record_id": 3824787,
|
||||
"submission_id": 541328,
|
||||
"decision_id": 580917,
|
||||
"status": "accepted",
|
||||
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
|
||||
"decision_date": null
|
||||
},
|
||||
{
|
||||
"external_id": "gong23a",
|
||||
"title": "CausalEdu: a real-world education dataset for temporal causal discovery and inference",
|
||||
"authors_json": "[\"Wenbo Gong\", \"Digory Smith\", \"Zichao Wang\", \"Craig Barton\", \"Simon Woodhead\", \"Nick Pawlowski\", \"Joel Jennings\", \"Cheng Zhang\"]",
|
||||
"paper_id": 3572634,
|
||||
"source_record_id": 3824792,
|
||||
"submission_id": 541333,
|
||||
"decision_id": 580922,
|
||||
"status": "accepted",
|
||||
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
|
||||
"decision_date": null
|
||||
},
|
||||
{
|
||||
"external_id": "munozbenitez23a",
|
||||
"title": "Synthetic Time Series: A Dataset for Causal Discovery",
|
||||
"authors_json": "[\"Julio Munoz-Benıtez\", \"L. Enrique Sucar\"]",
|
||||
"paper_id": 3572633,
|
||||
"source_record_id": 3824790,
|
||||
"submission_id": 541331,
|
||||
"decision_id": 580920,
|
||||
"status": "accepted",
|
||||
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
|
||||
"decision_date": null
|
||||
},
|
||||
{
|
||||
"external_id": "tec23a",
|
||||
"title": "SpaCE: The Spatial Confounding (Benchmarking) Environment",
|
||||
"authors_json": "[\"Mauricio Tec\", \"Ana Trisovic\", \"Michelle Audirac\", \"Francesca Dominici\"]",
|
||||
"paper_id": 1964409,
|
||||
"source_record_id": 3824791,
|
||||
"submission_id": 541332,
|
||||
"decision_id": 580921,
|
||||
"status": "accepted",
|
||||
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
|
||||
"decision_date": null
|
||||
},
|
||||
{
|
||||
"external_id": "xu23a",
|
||||
"title": "Causal Benchmark Based on Disentangled Image Dataset",
|
||||
"authors_json": "[\"Liyuan Xu\", \"Arthur Gretton\"]",
|
||||
"paper_id": 3572632,
|
||||
"source_record_id": 3824789,
|
||||
"submission_id": 541330,
|
||||
"decision_id": 580919,
|
||||
"status": "accepted",
|
||||
"evidence_url": "https://www.cclear.cc/2023/AcceptedDatasets",
|
||||
"decision_date": null
|
||||
}
|
||||
],
|
||||
"source": "clear2023_dataset_roster",
|
||||
"edition_id": 104179,
|
||||
"roster": "https://www.cclear.cc/2023/AcceptedDatasets",
|
||||
"bundle_sha256": "d5b9a0cda61581da679b1de7180da74dcc4d70625c984a401b10820083719a76",
|
||||
"original_main_bindings_preserved": 39,
|
||||
"prior_space_links_preserved": 2,
|
||||
"roster_entries": [
|
||||
{
|
||||
"entry_id": "chen23a",
|
||||
"title": "The Structurally Complex with Additive Parent Causality (SCARY) Dataset",
|
||||
"authors": [
|
||||
"Jarry Chen",
|
||||
"Haytham M. Fayek"
|
||||
],
|
||||
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/chen23a.pdf"
|
||||
},
|
||||
{
|
||||
"entry_id": "bizeul23a",
|
||||
"title": "3DIdentBox: A Toolbox for Identifiability Benchmarking",
|
||||
"authors": [
|
||||
"Alice Bizeul",
|
||||
"Imant Daunhawer",
|
||||
"Emanuele Palumbo",
|
||||
"Bernhard Schölkopf",
|
||||
"Alexander Marx",
|
||||
"Julia E. Vogt"
|
||||
],
|
||||
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/bizeul23a.pdf"
|
||||
},
|
||||
{
|
||||
"entry_id": "xu23a",
|
||||
"title": "Causal Benchmark Based on Disentangled Image Dataset",
|
||||
"authors": [
|
||||
"Liyuan Xu",
|
||||
"Arthur Gretton"
|
||||
],
|
||||
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/xu23a.pdf"
|
||||
},
|
||||
{
|
||||
"entry_id": "munozbenitez23a",
|
||||
"title": "Synthetic Time Series: A Dataset for Causal Discovery",
|
||||
"authors": [
|
||||
"Julio Munoz-Benıtez",
|
||||
"L. Enrique Sucar"
|
||||
],
|
||||
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/munozbenitez23a.pdf"
|
||||
},
|
||||
{
|
||||
"entry_id": "tec23a",
|
||||
"title": "SpaCE: The Spatial Confounding (Benchmarking) Environment",
|
||||
"authors": [
|
||||
"Mauricio Tec",
|
||||
"Ana Trisovic",
|
||||
"Michelle Audirac",
|
||||
"Francesca Dominici"
|
||||
],
|
||||
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/tec23a.pdf"
|
||||
},
|
||||
{
|
||||
"entry_id": "gong23a",
|
||||
"title": "CausalEdu: a real-world education dataset for temporal causal discovery and inference",
|
||||
"authors": [
|
||||
"Wenbo Gong",
|
||||
"Digory Smith",
|
||||
"Zichao Wang",
|
||||
"Craig Barton",
|
||||
"Simon Woodhead",
|
||||
"Nick Pawlowski",
|
||||
"Joel Jennings",
|
||||
"Cheng Zhang"
|
||||
],
|
||||
"programme_link": "https://www.cclear.cc/2023/AcceptedDatasets/gong23a.pdf"
|
||||
}
|
||||
],
|
||||
"roster_bundle_url": "https://www.cclear.cc/static/js/main.901abbee.chunk.js",
|
||||
"normal_query_verification": {
|
||||
"standard_query_verified": [
|
||||
580918,
|
||||
580917,
|
||||
580922,
|
||||
580920,
|
||||
580921,
|
||||
580919
|
||||
],
|
||||
"roster_count": 6
|
||||
},
|
||||
"space_identity_author_source": "https://anatrisovic.com/docs/atrisovic-cv.pdf",
|
||||
"provisional_work_ids": [
|
||||
3572631,
|
||||
3572632,
|
||||
3572633,
|
||||
3572634
|
||||
],
|
||||
"abstract_coverage": "The six source records have no roster-supplied abstracts. Existing canonical abstracts of linked works are retained.",
|
||||
"public_negative_decision_coverage": "The official accepted dataset roster does not publish a negative submission list. No negative decision was imported.",
|
||||
"deduplication_status": "Four provisional works await cross-source near-title review. The previous unbounded name query ended on timeout without results."
|
||||
}
|
||||
@@ -9,8 +9,23 @@ paper-atlas collect eccv2026-publications \
|
||||
--out /data/paper-index/collected/eccv-publisher-metadata
|
||||
```
|
||||
|
||||
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。
|
||||
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。数字上标的两种数学定界符可按同一标题核对,章节标题和目录原文分别保存。已有输出目录需另查回执或换新空目录后运行。
|
||||
|
||||
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
||||
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。单名署名须由同页的结构化作者列表按顺序确认,确认后按原名保存为字符串,引用元数据仍保留原文。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
||||
|
||||
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
||||
|
||||
## 已有节目投稿的出版证据
|
||||
|
||||
先在只读快照上复核节目 ID、章节 DOI、标题、完整有序作者、摘要版本与当前投稿历史。保存计划和原始证据,计划应列出每条出版观察、保留的论文与投稿 ID、节目来源和决定 ID、当前与目标届次及完整历史包的 SHA-256。作者或版本仍有疑点的条目继续留在待核清单。
|
||||
|
||||
```sh
|
||||
paper-atlas maintain eccv-publication-reconciliation \
|
||||
--pg-service <writer-service> --pg-schema research \
|
||||
--plan <reviewed-plan.json> --plan-sha256 <reviewed-plan-sha256> \
|
||||
--springer-previous-count <last-verified-count>
|
||||
```
|
||||
|
||||
命令接收 `readonly_same_submission_reconciliation_plan` 格式的已复核计划。它会在同一事务内核对实时历史和来源覆盖,把原节目投稿归入主会,追加出版来源与观测决定,并分别调整节目与 Springer 的覆盖数。节目主会覆盖保持 partial,因为其目录的主会分母尚未独立核定。随后通过只读连接核验投稿、决定及覆盖,保存计划和回执。相同计划只允许成功提交一次。若执行中失败,先查同一计划的提交标记、覆盖和投稿来源,再决定是否重新执行。
|
||||
|
||||
旧节目快照的全量未分类 manifest 无法刷新已有逐条更正的投稿。继续刷新时需按已复核轨道分别整理输出并生成 manifest。
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# KDD 2026 官方论文目录
|
||||
|
||||
从 [KDD 2026 论文目录](https://kdd2026.kdd.org/papers/) 保存公开 HTML,记录其 SHA-256,在已有目录中指定新的输出文件。
|
||||
|
||||
```sh
|
||||
paper-atlas collect kdd2026-program \
|
||||
--snapshot kdd2026-papers.html \
|
||||
--snapshot-sha256 <snapshot-sha256> \
|
||||
--out kdd2026-programme.json
|
||||
```
|
||||
|
||||
采集器只读取已保存的页面数据,逐项输出 DOI、官网标题、投稿周期、轨道和行哈希。完全相同的重复条目保留出现次数,冲突记录中止解析并等待人工复核。输出用于与来源和库内 DOI 核对,不能直接交给记录导入命令。
|
||||
|
||||
已核对的官网快照有 1,415 条目录记录和 1,408 个唯一 DOI。7 条重复记录的周期、轨道、标题和作者文本一致。February Cycle 列出 Research、Applied Data Science、Datasets and Benchmarks、AI for Sciences、Blue Sky Ideas 五轨。July Cycle 列出前三轨。官网列出的演讲、海报或扩展摘要需要出版方记录确认归档状态。[AI for Sciences 投稿及出版规则](https://kdd2026.kdd.org/ai4sciences-track-call-for-papers/)
|
||||
|
||||
Applied Data Science 有一条官网标注撤回并更换投稿 ID 的记录,旧链接为 `10.1145/3770855.3818478`。保留官网原文和日期字面值。原始投稿 ID 仍待公开证据核实。arXiv 当前指向 `10.1145/3770855.3828560`,这个 DOI 已属于同一份已有作品。两份来源分别记录,旧链接不作为新增作品标识。[arXiv 当前记录](https://arxiv.org/abs/2606.08923)
|
||||
|
||||
补分轨时按 DOI 检查库内已有投稿,先处理重复或缺少作者的记录。官网对录用材料的公开承诺适用于各轨道规定的范围,逐篇拒稿、初筛拒稿和其他撤回仍需公开决定证据。[Research](https://kdd2026.kdd.org/research-track-call-for-papers/) · [Applied Data Science](https://kdd2026.kdd.org/applied-data-science-ads-track-call-for-papers/) · [Datasets and Benchmarks](https://kdd2026.kdd.org/datasets-and-benchmarks-track-call-for-papers/) · [Blue Sky Ideas](https://kdd2026.kdd.org/kdd-2026-blue-sky-ideas-track-call-for-papers/)
|
||||
@@ -48,6 +48,8 @@ paper-atlas maintain source-revisions \
|
||||
|
||||
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
|
||||
|
||||
源快照和计划各自在 UTF-8 JSON 达到 1 MiB 时由客户端压缩后写入,避免数据库在接收大参数时出现内存峰值。读取兼容既有纯 JSON 和带 `gzip+base64:v1:` 前缀的压缩值,完整内容和计划哈希均可按原口径核验。使用该表执行写入或刷新时,运行时需要支持两种存储格式。
|
||||
|
||||
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
|
||||
|
||||
## 后续刷新及限制
|
||||
|
||||
@@ -173,34 +173,40 @@ def find_candidates(connection, records):
|
||||
records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
|
||||
|
||||
|
||||
def _named_rows(connection, sql, params):
|
||||
cursor = connection.execute(sql, params)
|
||||
columns = [field[0] for field in cursor.description]
|
||||
return [{name: row[i] for i, name in enumerate(columns)} for row in cursor]
|
||||
|
||||
|
||||
def entity_packet(connection, paper_id):
|
||||
paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone()
|
||||
if paper is None:
|
||||
papers = _named_rows(connection, 'SELECT * FROM papers WHERE id=?', (paper_id,))
|
||||
if not papers:
|
||||
raise RuntimeError('Candidate disappeared from read-only snapshot')
|
||||
sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source
|
||||
sources = _named_rows(connection, '''SELECT r.*,s.name AS source
|
||||
FROM source_records r JOIN sources s ON s.id=r.source_id
|
||||
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))]
|
||||
attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
|
||||
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))
|
||||
attempts = _named_rows(connection, '''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
|
||||
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
||||
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))]
|
||||
packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute(
|
||||
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))],
|
||||
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))
|
||||
packet = {'paper': papers[0], 'identifiers': _named_rows(connection,
|
||||
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,)),
|
||||
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
|
||||
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
|
||||
for attempt in packet['attempts']:
|
||||
observations = [dict(r) for r in connection.execute(
|
||||
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))]
|
||||
observations = _named_rows(connection,
|
||||
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))
|
||||
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
|
||||
provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source
|
||||
provenance = _named_rows(connection, '''SELECT ss.*,r.external_id,s.name AS source
|
||||
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
|
||||
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
|
||||
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))]
|
||||
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))
|
||||
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
|
||||
for decision in attempt['decisions']:
|
||||
provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source
|
||||
provenance = _named_rows(connection, '''SELECT ds.*,r.external_id,s.name AS source
|
||||
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
|
||||
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
|
||||
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))]
|
||||
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))
|
||||
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
|
||||
return packet
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ COMMANDS = {
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','kdd2026-program'): ('paper_atlas.collectors.kdd2026', {}, '核对 KDD 2026 两个投稿周期、分轨及撤回更换标注'),
|
||||
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
@@ -49,6 +50,7 @@ COMMANDS = {
|
||||
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
||||
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
||||
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
||||
('maintain','eccv-publication-reconciliation'): ('paper_atlas.ingestion.publication_reconciliation', {}, '向既有 ECCV 2026 投稿附加已复核的正式出版证据'),
|
||||
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
|
||||
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
||||
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
||||
|
||||
@@ -102,10 +102,20 @@ def publisher_inventory(root):
|
||||
return entries
|
||||
|
||||
|
||||
def title_for_comparison(title):
|
||||
# Springer may render a numeric superscript with inline or display math delimiters.
|
||||
return clean(re.sub(r'\\\((\^[0-9]+)\\\)|\$\$(\^[0-9]+)\$\$',
|
||||
lambda match: match[1] or match[2], title))
|
||||
|
||||
|
||||
def publisher_row(chapter, toc, evidence):
|
||||
doi = chapter['doi']
|
||||
if clean(chapter['title']) != toc['title']:
|
||||
literal_title_differs = clean(chapter['title']) != toc['title']
|
||||
if literal_title_differs and title_for_comparison(chapter['title']) != title_for_comparison(toc['title']):
|
||||
raise ValueError('Chapter title differs from Springer contents, review the version')
|
||||
contents_evidence = {'url': toc['evidence_url'], 'sha256': toc['evidence_sha256']}
|
||||
if literal_title_differs:
|
||||
contents_evidence['title_literal'] = toc['title']
|
||||
url = ORIGIN + '/chapter/' + doi
|
||||
return {
|
||||
'source': SOURCE, 'external_id': doi, 'url': url,
|
||||
@@ -120,7 +130,7 @@ def publisher_row(chapter, toc, evidence):
|
||||
'book_title': BOOK_TITLE, 'book_doi': toc['book_doi'], 'lncs_volume': toc['lncs_volume'],
|
||||
'authors_citation_raw': chapter['authors_citation_raw'], 'first_page': chapter['first_page'],
|
||||
'last_page': chapter['last_page'], 'publisher_chapter_evidence': evidence,
|
||||
'publisher_contents_evidence': {'url': toc['evidence_url'], 'sha256': toc['evidence_sha256']},
|
||||
'publisher_contents_evidence': contents_evidence,
|
||||
'scope': 'Refereed conference proceedings, LNCS 17001–17078.',
|
||||
'original_decision_signature_verified': False,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Inventory KDD 2026 programme DOI, cycle and track observations."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from hashlib import sha256
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
SOURCE_URL = 'https://kdd2026.kdd.org/papers/'
|
||||
CYCLES = {1: 'july', 2: 'february'}
|
||||
TRACKS = {
|
||||
'rtp': 'research', 'ads': 'applied_data_science', 'dtb': 'datasets_and_benchmarks',
|
||||
'ais': 'ai_for_sciences', 'bsi': 'blue_sky_ideas',
|
||||
}
|
||||
DOI_URL = re.compile(r'https://doi\.org/(10\.1145/\d+\.\d+)')
|
||||
WITHDRAWN = re.compile(
|
||||
r'^\*\*WITHDRAWN/New SUBMISSION ID for Authorship Order Change (\d{1,2}/\d{1,2}/26) -- (.+)$'
|
||||
)
|
||||
OTHER_STATUS = re.compile(r'^[*\[(!#\s]*(?:withdrawn|rejected|desk[\s_-]+rejected|retracted|cancell?ed)\b', re.I)
|
||||
|
||||
|
||||
class ProgrammeScripts(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.current = None
|
||||
self.scripts = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == 'script' and 'src' not in dict(attrs):
|
||||
self.current = []
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.current is not None:
|
||||
self.current.append(data)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == 'script' and self.current is not None:
|
||||
self.scripts.append(''.join(self.current))
|
||||
self.current = None
|
||||
|
||||
|
||||
def cycle_rows(scripts, cycle):
|
||||
pattern = re.compile(r'\bconst\s+cycle' + str(cycle) + r'Papers\s*=\s*(\[)')
|
||||
matches = [(code, match) for code in scripts for match in pattern.finditer(code)]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f'Expected one KDD programme array for cycle {cycle}')
|
||||
code, match = matches[0]
|
||||
body = code[match.start(1):]
|
||||
rows, consumed = json.JSONDecoder().raw_decode(body)
|
||||
if not rows or not body[consumed:].lstrip().startswith('.map((paper, index)'):
|
||||
raise ValueError(f'Unexpected KDD programme array boundary for cycle {cycle}')
|
||||
return rows
|
||||
|
||||
|
||||
def inventory(raw):
|
||||
parser = ProgrammeScripts()
|
||||
parser.feed(raw.decode('utf-8'))
|
||||
parser.close()
|
||||
records = {}
|
||||
source_counts = {}
|
||||
for cycle, cycle_name in CYCLES.items():
|
||||
rows = cycle_rows(parser.scripts, cycle)
|
||||
source_counts[cycle_name] = Counter()
|
||||
for row in rows:
|
||||
if (not isinstance(row, dict) or row.keys() != {'track', 'title', 'url', 'authors'}
|
||||
or not all(isinstance(value, str) for value in row.values())
|
||||
or row['track'] not in TRACKS or not row['title'].strip()):
|
||||
raise ValueError('Unexpected KDD programme row')
|
||||
match = DOI_URL.fullmatch(row['url'])
|
||||
if not match:
|
||||
raise ValueError('Invalid DOI link in KDD programme')
|
||||
doi = match[1]
|
||||
track = TRACKS[row['track']]
|
||||
source_counts[cycle_name][track] += 1
|
||||
row_hash = sha256(json.dumps(row, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
if doi in records:
|
||||
entry = records[doi]
|
||||
if (entry['cycle'], entry['source_row_sha256']) != (cycle_name, row_hash):
|
||||
raise ValueError('Conflicting duplicate DOI in KDD programme: ' + doi)
|
||||
entry['occurrences'] += 1
|
||||
continue
|
||||
status = 'listed'
|
||||
withdrawn = WITHDRAWN.fullmatch(row['title'])
|
||||
if withdrawn:
|
||||
status = 'withdrawn_with_replacement_mentioned'
|
||||
elif OTHER_STATUS.match(row['title']):
|
||||
raise ValueError('Unrecognized status annotation in KDD programme: ' + doi)
|
||||
records[doi] = {
|
||||
'doi': doi, 'doi_url': row['url'], 'source_title': row['title'],
|
||||
'title': withdrawn[2] if withdrawn else row['title'],
|
||||
'cycle': cycle_name, 'track': track, 'programme_status': status,
|
||||
'authors_listed': bool(row['authors'].strip()), 'occurrences': 1,
|
||||
'source_row_sha256': row_hash,
|
||||
}
|
||||
if withdrawn:
|
||||
records[doi]['source_withdrawal_date_literal'] = withdrawn[1]
|
||||
ordered = [records[doi] for doi in sorted(records)]
|
||||
return ordered, source_counts
|
||||
|
||||
|
||||
def collect(snapshot, expected_sha256, out):
|
||||
snapshot = Path(snapshot)
|
||||
with snapshot.open('rb') as stream:
|
||||
raw = stream.read(4_000_001)
|
||||
if len(raw) > 4_000_000 or sha256(raw).hexdigest() != expected_sha256:
|
||||
raise ValueError('KDD programme snapshot hash or size differs')
|
||||
rows, cycle_counts = inventory(raw)
|
||||
result = {
|
||||
'source': SOURCE_URL, 'snapshot_sha256': expected_sha256,
|
||||
'source_rows_by_cycle_and_track': cycle_counts, 'unique_dois': len(rows),
|
||||
'identical_repeat_rows': sum(row['occurrences'] - 1 for row in rows),
|
||||
'programme_status_counts': Counter(row['programme_status'] for row in rows),
|
||||
'records': rows, 'database_written': False,
|
||||
}
|
||||
with Path(out).open('x', encoding='utf-8') as stream:
|
||||
json.dump(result, stream, ensure_ascii=False, indent=2)
|
||||
stream.write('\n')
|
||||
return result
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog)
|
||||
parser.add_argument('--snapshot', required=True)
|
||||
parser.add_argument('--snapshot-sha256', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
result = collect(args.snapshot, args.snapshot_sha256, args.out)
|
||||
print(json.dumps({k: result[k] for k in ('unique_dois', 'identical_repeat_rows', 'programme_status_counts')}))
|
||||
@@ -1,4 +1,6 @@
|
||||
"""Versioned publication-source scopes and guards against stale source refreshes."""
|
||||
import base64
|
||||
import gzip
|
||||
import json
|
||||
import re
|
||||
|
||||
@@ -6,6 +8,24 @@ TABLE = 'source_scope_revisions'
|
||||
MARKER = 'source_scope_revisions_v1'
|
||||
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
|
||||
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
|
||||
COMPRESSED_PREFIX = 'gzip+base64:v1:'
|
||||
COMPRESSION_THRESHOLD = 1024 * 1024
|
||||
|
||||
|
||||
def encode_payload(value):
|
||||
"""Compress large revision payloads before sending them to the database."""
|
||||
text = json.dumps(value, ensure_ascii=False, separators=(',', ':'))
|
||||
raw = text.encode('utf-8')
|
||||
if len(raw) < COMPRESSION_THRESHOLD:
|
||||
return text
|
||||
return COMPRESSED_PREFIX + base64.b64encode(gzip.compress(raw, compresslevel=1, mtime=0)).decode('ascii')
|
||||
|
||||
|
||||
def decode_payload(text):
|
||||
if text.startswith(COMPRESSED_PREFIX):
|
||||
packed = base64.b64decode(text[len(COMPRESSED_PREFIX):], validate=True)
|
||||
return json.loads(gzip.decompress(packed))
|
||||
return json.loads(text)
|
||||
|
||||
|
||||
def available(c):
|
||||
@@ -60,7 +80,7 @@ def current_rows(c, source_id, cache):
|
||||
records = {}; scopes = {}
|
||||
for signature in signatures:
|
||||
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
|
||||
scopes[signature] = json.loads(raw)
|
||||
scopes[signature] = decode_payload(raw)
|
||||
for row in scopes[signature]:
|
||||
ext = row['external_id']
|
||||
if ext in records and records[ext] != row:
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Attach reviewed ECCV proceedings evidence to an existing programme attempt."""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.audits.source_identity import entity_packet
|
||||
from paper_atlas.collectors.eccv2026 import SOURCE as PROGRAMME
|
||||
from paper_atlas.collectors.eccv2026_publications import SOURCE as PUBLISHER
|
||||
from paper_atlas.db.backend import database_arguments, open_store
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
from paper_atlas.db.submissions import identities_for_row, timestamp
|
||||
from paper_atlas.ingestion.edition_tracks import curation
|
||||
from paper_atlas.ingestion.records import source_record_count
|
||||
|
||||
MARKER = 'eccv_publication_reconciliation_v1:'
|
||||
PUBLISHER_COUNT = 2835
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def packet_digest(packet):
|
||||
return sha256(json.dumps(packet, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
|
||||
|
||||
def validate_plan(plan):
|
||||
require(plan.get('status') == 'readonly_same_submission_reconciliation_plan'
|
||||
and plan.get('database_written') is False, 'A reviewed read-only plan is required')
|
||||
rows = plan['plans']
|
||||
require(0 < len(rows) <= 1000 and len(rows) == plan['guarded_same_submission_plans'],
|
||||
'Unexpected approved plan size')
|
||||
for key in ('doi', 'retained_paper_id', 'retained_submission_id', 'retained_programme_record_id'):
|
||||
require(len({entry[key] for entry in rows}) == len(rows), 'Repeated reviewed ' + key)
|
||||
for entry in rows:
|
||||
doi = entry['doi']
|
||||
row = entry['source_row']
|
||||
url = 'https://link.springer.com/chapter/' + doi
|
||||
evidence = row['metadata']['publisher_chapter_evidence']
|
||||
require(re.fullmatch(r'10\.1007/978-3-032-\d{5}-[0-9Xx]_[1-9]\d*', doi)
|
||||
and (row['source'], row['external_id'], row['url'], row['evidence_url']) ==
|
||||
(PUBLISHER, doi, url, url)
|
||||
and (row['venue'], row['year'], row['track'], row['status']) ==
|
||||
('ECCV', 2026, 'main', 'accepted')
|
||||
and row['identifiers'] == {'doi': doi}
|
||||
and identities_for_row(row) == [('publication_doi', doi)]
|
||||
and isinstance(row['abstract'], str) and row['abstract'].strip()
|
||||
and re.fullmatch(r'[0-9a-f]{64}', evidence['sha256'])
|
||||
and re.fullmatch(r'[0-9a-f]{64}', entry['before_packet_sha256']),
|
||||
'Invalid reviewed publisher observation')
|
||||
return rows
|
||||
|
||||
|
||||
def existing_coverage(conn, edition, source):
|
||||
rows = conn.execute('''SELECT list_url,status,expected_count,imported_count,cursor,last_success_at
|
||||
FROM edition_sources WHERE edition_id=? AND source_id=?''', (edition, source)).fetchall()
|
||||
require(len(rows) <= 1, 'Source coverage is ambiguous')
|
||||
actual = source_record_count(conn, source, edition)
|
||||
if not rows:
|
||||
require(actual == 0, 'Linked records have no coverage listing')
|
||||
return None
|
||||
row = rows[0]
|
||||
require(row[1] == 'partial' and row[3] == actual, 'Coverage differs from linked records')
|
||||
return tuple(row)
|
||||
|
||||
|
||||
def reconcile_attempt(store, entry, original, target, venue_id, event_key, venue_names, publisher, signature):
|
||||
conn = store.conn
|
||||
doi = entry['doi']
|
||||
canonical_doi = doi.lower()
|
||||
paper = entry['retained_paper_id']
|
||||
submission = entry['retained_submission_id']
|
||||
programme_record = entry['retained_programme_record_id']
|
||||
row = entry['source_row']
|
||||
before = entity_packet(conn, paper)
|
||||
require(packet_digest(before) == entry['before_packet_sha256'], 'Reviewed paper history changed: ' + doi)
|
||||
require(not before['source_records_truncated'] and not before['attempts_truncated']
|
||||
and all(not attempt['decisions_truncated'] and not attempt['sources_truncated']
|
||||
and all(not decision['sources_truncated'] for decision in attempt['decisions'])
|
||||
for attempt in before['attempts']), 'Reviewed paper history exceeds verification limits')
|
||||
same_event = [a for a in before['attempts'] if a['venue'].casefold() == 'eccv' and a['year'] == 2026]
|
||||
require(len(same_event) == 1, 'Paper has multiple or missing ECCV 2026 attempts')
|
||||
attempt = same_event[0]
|
||||
require((attempt['id'], attempt['edition_id'], attempt['source_record_id']) ==
|
||||
(submission, original, programme_record)
|
||||
and [(d['id'], d['status']) for d in attempt['decisions']] ==
|
||||
[(entry['retained_programme_decision_id'], 'accepted')]
|
||||
and [(s['source_record_id'], s['source']) for s in attempt['sources']] ==
|
||||
[(programme_record, PROGRAMME)], 'Original programme attempt differs')
|
||||
programme = next(record for record in before['source_records'] if record['id'] == programme_record)
|
||||
metadata = json.loads(programme['metadata_json'])
|
||||
require(KEY not in metadata and attempt['source_submission_id'] == programme['external_id'],
|
||||
'Programme observation already curated or ambiguous')
|
||||
scope = curation({
|
||||
'external_id': programme['external_id'], 'venue_names': venue_names, 'from_edition_id': original,
|
||||
'year': 2026, 'from_track': 'program_unclassified', 'event_key': event_key,
|
||||
'venue_id': venue_id, 'track': 'main', 'section': 'published ECCV refereed proceedings',
|
||||
'evidence_url': row['evidence_url'], 'evidence_sha256': row['metadata']['publisher_chapter_evidence']['sha256'],
|
||||
}, target, signature)
|
||||
require(not conn.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=?', (submission,)).fetchone()
|
||||
and not conn.execute("SELECT 1 FROM submission_identifiers WHERE edition_id=? AND scheme='publication_doi' AND value=?",
|
||||
(target, doi)).fetchone()
|
||||
and not conn.execute('SELECT 1 FROM source_records WHERE source_id=? AND external_id=?',
|
||||
(publisher, doi)).fetchone(), 'Publication observation or submission identity already exists')
|
||||
require(not conn.execute('SELECT 1 FROM identifiers WHERE scheme=? AND value=?',
|
||||
(PUBLISHER, doi)).fetchone(), 'Publisher identity already exists')
|
||||
owner = conn.execute("SELECT paper_id FROM identifiers WHERE scheme='doi' AND value=?", (canonical_doi,)).fetchone()
|
||||
require(owner is None or owner[0] == paper, 'Publication DOI belongs to another paper')
|
||||
if owner is None:
|
||||
conn.execute("INSERT INTO identifiers(scheme,value,paper_id) VALUES('doi',?,?)", (canonical_doi, paper))
|
||||
before['identifiers'].append({'scheme': 'doi', 'value': canonical_doi, 'paper_id': paper})
|
||||
before['identifiers'].append({'scheme': PUBLISHER, 'value': doi, 'paper_id': paper})
|
||||
before['identifiers'].sort(key=lambda identifier: (identifier['scheme'], identifier['value']))
|
||||
require(conn.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',
|
||||
(target, submission, original)).rowcount == 1, 'Programme attempt changed')
|
||||
conn.execute("INSERT INTO submission_identifiers VALUES(?,'publication_doi',?,?)", (target, doi, submission))
|
||||
metadata[KEY] = [scope]
|
||||
programme['metadata_json'] = json.dumps(metadata, ensure_ascii=False)
|
||||
conn.execute('UPDATE source_records SET metadata_json=? WHERE id=?',
|
||||
(programme['metadata_json'], programme_record))
|
||||
actual_paper, record = store.record(PUBLISHER, doi, row['url'], row['title'], row['abstract'],
|
||||
row['authors'], row['identifiers'], row['metadata'])
|
||||
require(actual_paper == paper, 'Unexpected canonical paper')
|
||||
store.record_tags(paper, record, PUBLISHER, row['tags'])
|
||||
actual_edition = store.decision(paper, record, 'ECCV', 2026, 'main', doi, row['evidence_url'],
|
||||
'accepted', row['raw_decision'], identities=identities_for_row(row),
|
||||
observed_at=row['observed_at'], event_key=event_key)
|
||||
require(actual_edition == target, 'Unexpected target edition')
|
||||
|
||||
after = entity_packet(conn, paper)
|
||||
published = [r for r in after['source_records'] if r['id'] == record]
|
||||
require(len(published) == 1 and published[0]['external_id'] == doi
|
||||
and published[0]['abstract'] == row['abstract'], 'Publication source is missing or changed')
|
||||
after['source_records'].remove(published[0])
|
||||
moved = next(a for a in after['attempts'] if a['id'] == submission)
|
||||
attached = [s for s in moved['sources'] if s['source_record_id'] == record]
|
||||
observed = [d for d in moved['decisions'] if any(s['source_record_id'] == record for s in d['sources'])]
|
||||
require(len(attached) == len(observed) == 1 and attached[0]['source_submission_id'] == doi
|
||||
and observed[0]['status'] == 'accepted' and observed[0]['evidence_url'] == row['evidence_url']
|
||||
and len(observed[0]['sources']) == 1, 'Publisher evidence did not attach to the existing submission')
|
||||
moved['sources'].remove(attached[0])
|
||||
moved['decisions'].remove(observed[0])
|
||||
attempt.update(edition_id=target, track='main')
|
||||
require(before == after, 'Existing paper or submission history changed')
|
||||
|
||||
|
||||
def apply(store, plan, signature, previous_count):
|
||||
rows = validate_plan(plan)
|
||||
conn = store.conn
|
||||
marker = MARKER + signature
|
||||
require(not conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (marker,)).fetchone(),
|
||||
'Reconciliation already applied')
|
||||
edition_rows = conn.execute('''SELECT e.track,e.id,e.venue_id,e.event_key
|
||||
FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym='ECCV' AND e.year=2026
|
||||
AND e.track IN ('main','program_unclassified')''').fetchall()
|
||||
require(len(edition_rows) == 2 and {r[0] for r in edition_rows} == {'main', 'program_unclassified'},
|
||||
'Expected two unambiguous reviewed editions')
|
||||
editions = {r[0]: (r[1], r[2], r[3]) for r in edition_rows}
|
||||
original, venue, event = editions['program_unclassified']
|
||||
target, target_venue, target_event = editions['main']
|
||||
require(venue == target_venue and event == target_event
|
||||
and all((r['move_from_edition_id'], r['move_to_edition_id']) == (original, target) for r in rows),
|
||||
'Reviewed edition identity differs')
|
||||
names = ['ECCV', *(r[0] for r in conn.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (venue,)))]
|
||||
sources = {r[0]: r[1] for r in conn.execute('SELECT name,id FROM sources WHERE name IN (?,?)', (PROGRAMME, PUBLISHER))}
|
||||
programme, publisher = sources[PROGRAMME], sources[PUBLISHER]
|
||||
old = existing_coverage(conn, original, programme)
|
||||
main = existing_coverage(conn, target, programme)
|
||||
published = existing_coverage(conn, target, publisher)
|
||||
require(old is not None and old[3] >= len(rows) and (main is None or main[0] == old[0])
|
||||
and published is not None and published[2:4] == (PUBLISHER_COUNT, previous_count)
|
||||
and previous_count + len(rows) <= PUBLISHER_COUNT, 'Reviewed coverage parent differs')
|
||||
for entry in rows:
|
||||
reconcile_attempt(store, entry, original, target, venue, event, names, publisher, signature)
|
||||
|
||||
remaining = old[3] - len(rows)
|
||||
imported = previous_count + len(rows)
|
||||
main_count = (main[3] if main else 0) + len(rows)
|
||||
for edition, source, expected in ((original, programme, remaining),
|
||||
(target, programme, main_count), (target, publisher, imported)):
|
||||
require(source_record_count(conn, source, edition) == expected, 'Reconciled coverage differs from linked records')
|
||||
conn.execute('UPDATE edition_sources SET imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
|
||||
(remaining, original, programme, old[0]))
|
||||
conn.execute('''INSERT INTO edition_sources
|
||||
(edition_id,source_id,list_url,status,imported_count,cursor,last_success_at) VALUES(?,?,?,'partial',?,?,?)
|
||||
ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET imported_count=excluded.imported_count''',
|
||||
(target, programme, old[0], main_count, old[4], old[5]))
|
||||
conn.execute('''UPDATE edition_sources SET imported_count=?,last_success_at=?
|
||||
WHERE edition_id=? AND source_id=? AND list_url=?''', (imported, timestamp(), target, publisher, published[0]))
|
||||
conn.execute('INSERT INTO schema_migrations(name) VALUES(?)', (marker,))
|
||||
return {'status': 'applied', 'plan_sha256': signature, 'programme_main': main_count,
|
||||
'programme_unclassified': remaining, 'springer_main': imported, 'new_papers': 0, 'new_submissions': 0}
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog)
|
||||
database_arguments(parser)
|
||||
parser.add_argument('--plan', type=Path, required=True)
|
||||
parser.add_argument('--plan-sha256', required=True)
|
||||
parser.add_argument('--springer-previous-count', type=int, required=True)
|
||||
args = parser.parse_args(argv)
|
||||
raw = args.plan.read_bytes()
|
||||
signature = sha256(raw).hexdigest()
|
||||
require(signature == args.plan_sha256, 'Plan hash differs')
|
||||
plan = json.loads(raw)
|
||||
validate_plan(plan)
|
||||
store = open_store(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
||||
with closing(store.conn) as conn:
|
||||
try:
|
||||
if not getattr(conn, 'postgres', False):
|
||||
conn.commit()
|
||||
conn.execute('BEGIN IMMEDIATE')
|
||||
result = apply(store, plan, signature, args.springer_previous_count)
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
print(json.dumps(result), flush=True)
|
||||
@@ -203,7 +203,7 @@ def changes(c, inventory, before):
|
||||
def proposal(c, inventory, identity):
|
||||
before = capture(c, inventory)
|
||||
if before['parent_sha256']:
|
||||
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
|
||||
saved = versions.decode_payload(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
|
||||
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
|
||||
raise ValueError('Previous snapshot differs from active source version')
|
||||
records, coverage = changes(c, inventory, before)
|
||||
@@ -241,8 +241,8 @@ def verify(c, plan, signature):
|
||||
if after['coverage'] != expected_coverage:
|
||||
raise ValueError('Reconciled coverage differs; roll back')
|
||||
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
|
||||
if (not stored or json.loads(stored[0]) != plan['inventory']['rows']
|
||||
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
|
||||
if (not stored or versions.decode_payload(stored[0]) != plan['inventory']['rows']
|
||||
or versions.decode_payload(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
|
||||
raise ValueError('Versioned source or complete before-images changed')
|
||||
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
|
||||
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
|
||||
@@ -269,6 +269,9 @@ def apply_plan(c, plan):
|
||||
signature = digest(plan)
|
||||
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
|
||||
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
||||
inventory = plan['inventory']
|
||||
records_payload = versions.encode_payload(inventory['rows'])
|
||||
plan_payload = versions.encode_payload(plan)
|
||||
if getattr(c, 'postgres', False):
|
||||
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
|
||||
if proposal(c, plan['inventory'], plan['database']) != plan:
|
||||
@@ -284,10 +287,10 @@ def apply_plan(c, plan):
|
||||
before = row['before']
|
||||
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
|
||||
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
|
||||
inventory = plan['inventory']
|
||||
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
|
||||
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False)))
|
||||
inventory['list_url'], plan['before']['parent_sha256'],
|
||||
records_payload, plan_payload))
|
||||
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
||||
|
||||
|
||||
|
||||
@@ -54,6 +54,27 @@ def read_page(index, url):
|
||||
return raw, evidence
|
||||
|
||||
|
||||
def require_display_authors(nodes, authors):
|
||||
matching_article_lists = []
|
||||
for node in nodes:
|
||||
if node.tag != 'script' or node.attrs.get('type') != 'application/ld+json':
|
||||
continue
|
||||
try:
|
||||
article = json.loads(''.join(node.children))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(article, dict) or article.get('@type') != 'ScholarlyArticle':
|
||||
continue
|
||||
entries = article.get('author')
|
||||
if not isinstance(entries, list) or not all(
|
||||
isinstance(entry, dict) and isinstance(entry.get('name'), str) for entry in entries
|
||||
):
|
||||
raise ValueError('Unreviewed Springer article author list')
|
||||
matching_article_lists.append([clean(entry['name']) for entry in entries])
|
||||
if matching_article_lists != [authors]:
|
||||
raise ValueError('Single-name author requires matching Springer display author list')
|
||||
|
||||
|
||||
def parse_chapter(html, doi, *, book_title, publication_year):
|
||||
nodes = list(Document(html).root.walk())
|
||||
meta = defaultdict(list)
|
||||
@@ -73,12 +94,22 @@ def parse_chapter(html, doi, *, book_title, publication_year):
|
||||
raise ValueError('Missing named chapter authors')
|
||||
authors = []
|
||||
for raw in raw_authors:
|
||||
if ',' not in raw:
|
||||
name = clean(raw)
|
||||
if len(name.split()) != 1 or not any(character.isalpha() for character in name):
|
||||
raise ValueError('Unreviewed citation author format')
|
||||
authors.append(name)
|
||||
continue
|
||||
if raw.count(',') != 1:
|
||||
raise ValueError('Unreviewed citation author format')
|
||||
family, given = [clean(s) for s in raw.split(',', 1)]
|
||||
if not family or not given:
|
||||
raise ValueError('Incomplete citation author')
|
||||
authors.append({'given':given, 'family':family})
|
||||
if any(isinstance(author, str) for author in authors):
|
||||
display_names = [author if isinstance(author, str) else author['given']+' '+author['family']
|
||||
for author in authors]
|
||||
require_display_authors(nodes, display_names)
|
||||
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
||||
if len(abstract_nodes) > 1:
|
||||
raise ValueError('Ambiguous abstract section')
|
||||
|
||||
@@ -26,6 +26,10 @@ class Result:
|
||||
def rowcount(self):
|
||||
return self.cursor.rowcount
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
return self.cursor.description
|
||||
|
||||
|
||||
class WriteConnection:
|
||||
postgres = True
|
||||
|
||||
@@ -6,7 +6,7 @@ import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors.eccv2026_publications import BOOK_TITLE, SOURCE, collect
|
||||
from paper_atlas.collectors.eccv2026_publications import BOOK_TITLE, SOURCE, collect, publisher_row
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
|
||||
@@ -114,6 +114,19 @@ class ECCV2026PublicationsTests(unittest.TestCase):
|
||||
with self.assertRaises(FileExistsError):
|
||||
collect(root / 'out', **source)
|
||||
|
||||
def test_numeric_superscript_wrappers_preserve_both_title_literals(self):
|
||||
chapter = {'doi': FIRST_DOI, 'title': 'Example $$^2$$ : Formal title',
|
||||
'authors': [], 'abstract': 'Published abstract.', 'authors_citation_raw': [],
|
||||
'first_page': '1', 'last_page': '10'}
|
||||
toc = {'title': r'Example \(^2\) : Formal title', 'book_doi': FIRST_BOOK, 'lncs_volume': 17001,
|
||||
'evidence_url': 'https://link.springer.com/book/' + FIRST_BOOK, 'evidence_sha256': 'a' * 64}
|
||||
evidence = {'observed_at': '2026-11-18T00:00:00Z'}
|
||||
row = publisher_row(chapter, toc, evidence)
|
||||
self.assertEqual(row['title'], chapter['title'])
|
||||
self.assertEqual(row['metadata']['publisher_contents_evidence']['title_literal'], toc['title'])
|
||||
with self.assertRaisesRegex(ValueError, 'title differs'):
|
||||
publisher_row(chapter, {**toc, 'title': r'Example \(^3\) : Formal title'}, evidence)
|
||||
|
||||
def test_changed_contents_receipt_stops_before_output(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
|
||||
from paper_atlas.cli import main
|
||||
from paper_atlas.collectors.kdd2026 import inventory
|
||||
|
||||
|
||||
def row(number, track, title='Paper title', authors='Name (Affiliation)'):
|
||||
return {'track': track, 'title': title, 'url': f'https://doi.org/10.1145/3770855.{number}', 'authors': authors}
|
||||
|
||||
|
||||
def snapshot(july, february):
|
||||
arrays = [
|
||||
f'<script>const cycle{n}Papers = {json.dumps(rows)}.map((paper, index) => paper);</script>'
|
||||
for n, rows in ((1, july), (2, february))
|
||||
]
|
||||
return '\n'.join(arrays).encode()
|
||||
|
||||
|
||||
class KDD2026Tests(unittest.TestCase):
|
||||
def test_cli_preserves_withdrawal_marker_and_identical_occurrences(self):
|
||||
retained = row(1, 'rtp')
|
||||
withdrawn = row(2, 'ads', '**WITHDRAWN/New SUBMISSION ID for Authorship Order Change 6/24/26 -- A revised work')
|
||||
incomplete = row(3, 'ais', authors='')
|
||||
raw = snapshot([retained, retained], [withdrawn, incomplete])
|
||||
with TemporaryDirectory() as directory:
|
||||
source = Path(directory) / 'snapshot.html'
|
||||
source.write_bytes(raw)
|
||||
out = source.with_suffix('.json')
|
||||
args = ['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
|
||||
sha256(raw).hexdigest(), '--out', str(out)]
|
||||
main(args)
|
||||
report = json.loads(out.read_bytes())
|
||||
with self.assertRaises(FileExistsError):
|
||||
main(args)
|
||||
self.assertEqual(report['source_rows_by_cycle_and_track'], {
|
||||
'july': {'research': 2}, 'february': {'applied_data_science': 1, 'ai_for_sciences': 1},
|
||||
})
|
||||
self.assertEqual(report['unique_dois'], 3)
|
||||
self.assertEqual(report['identical_repeat_rows'], 1)
|
||||
kept, replaced, sparse = report['records']
|
||||
self.assertEqual(kept['occurrences'], 2)
|
||||
self.assertEqual(replaced['programme_status'], 'withdrawn_with_replacement_mentioned')
|
||||
self.assertEqual(replaced['title'], 'A revised work')
|
||||
self.assertEqual(replaced['source_title'], withdrawn['title'])
|
||||
self.assertEqual(replaced['source_withdrawal_date_literal'], '6/24/26')
|
||||
self.assertNotIn('submission_id', replaced)
|
||||
self.assertFalse(sparse['authors_listed'])
|
||||
self.assertFalse(report['database_written'])
|
||||
|
||||
def test_conflicting_duplicate_and_unknown_status_fail(self):
|
||||
with self.assertRaisesRegex(ValueError, 'Conflicting duplicate DOI'):
|
||||
inventory(snapshot([row(1, 'rtp')], [row(1, 'ads')]))
|
||||
with self.assertRaisesRegex(ValueError, 'Unrecognized status'):
|
||||
inventory(snapshot([row(1, 'rtp')], [row(2, 'ads', '***WITHDRAWN -- Unknown format')]))
|
||||
|
||||
def test_hash_mismatch_leaves_no_output(self):
|
||||
with TemporaryDirectory() as directory:
|
||||
source = Path(directory) / 'snapshot.html'
|
||||
source.write_bytes(snapshot([row(1, 'rtp')], [row(2, 'ads')]))
|
||||
out = source.with_suffix('.json')
|
||||
with self.assertRaisesRegex(ValueError, 'snapshot hash or size'):
|
||||
main(['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
|
||||
'0' * 64, '--out', str(out)])
|
||||
self.assertFalse(out.exists())
|
||||
@@ -0,0 +1,140 @@
|
||||
from contextlib import closing, redirect_stdout
|
||||
from hashlib import sha256
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from paper_atlas.audits.source_identity import entity_packet
|
||||
from paper_atlas.cli import main
|
||||
from paper_atlas.db.edition_scopes import resolve
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.publication_reconciliation import PROGRAMME, PUBLISHER, packet_digest
|
||||
from paper_atlas.postgres.client import ReadConnection, connect
|
||||
from paper_atlas.postgres.migration import migrate
|
||||
from paper_atlas.postgres.schema import quote
|
||||
|
||||
|
||||
class PublicationReconciliationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
directory = TemporaryDirectory()
|
||||
self.addCleanup(directory.cleanup)
|
||||
self.root = Path(directory.name)
|
||||
self.db = self.root / 'db.sqlite'
|
||||
self.store = Store(self.db)
|
||||
self.conn = self.store.conn
|
||||
self.addCleanup(self.conn.close)
|
||||
self.programme_url = 'https://eccv.ecva.net/Conferences/2026/AcceptedPapers'
|
||||
self.publisher_url = 'https://link.springer.com/conference/eccv'
|
||||
programme = self.store.source(PROGRAMME, self.programme_url)
|
||||
publisher = self.store.source(PUBLISHER, self.publisher_url)
|
||||
self.entries = [self.programme_attempt(str(i)) for i in range(1, 4)]
|
||||
title = 'A distinct chapter already integrated in the main track'
|
||||
paper, record = self.store.record(PUBLISHER, 'prior', self.publisher_url, title, 'Text', [])
|
||||
self.target = self.store.accepted(paper, record, 'ECCV', 2026, 'main', 'prior', self.publisher_url)
|
||||
self.original = self.entries[0]['move_from_edition_id']
|
||||
for entry in self.entries:
|
||||
entry['move_to_edition_id'] = self.target
|
||||
self.conn.executemany('''INSERT INTO edition_sources
|
||||
(edition_id,source_id,list_url,status,expected_count,imported_count) VALUES(?,?,?,'partial',?,?)''',
|
||||
[(self.original, programme, self.programme_url, 2834, 3),
|
||||
(self.target, publisher, self.publisher_url, 2835, 1)])
|
||||
self.conn.commit()
|
||||
|
||||
def programme_attempt(self, number):
|
||||
title = 'A programme observation for continuing research ' + number
|
||||
url = self.programme_url + '/' + number
|
||||
authors = ['Ada Example', 'Sam Example']
|
||||
paper, record = self.store.record(PROGRAMME, number, url, title, 'Programme text', authors)
|
||||
original = self.store.accepted(paper, record, 'ECCV', 2026, 'program_unclassified', number, url)
|
||||
other, old_record = self.store.record('openreview', 'test-' + number, url, title, 'Programme text', authors)
|
||||
self.assertEqual(other, paper)
|
||||
self.store.decision(paper, old_record, 'ICLR', 2025, 'main', 'test-' + number, url, 'rejected', 'Reject')
|
||||
packet = entity_packet(self.conn, paper)
|
||||
attempt = next(row for row in packet['attempts'] if row['source_record_id'] == record)
|
||||
doi = '10.1007/978-3-032-00000-1_' + number
|
||||
chapter = 'https://link.springer.com/chapter/' + doi
|
||||
row = {'source': PUBLISHER, 'external_id': doi, 'url': chapter, 'evidence_url': chapter,
|
||||
'venue': 'ECCV', 'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||
'title': title, 'authors': authors, 'abstract': 'Publisher text',
|
||||
'observed_at': '2026-11-18T00:00:00Z', 'raw_decision': 'Publication observation',
|
||||
'identifiers': {'doi': doi},
|
||||
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||
'metadata': {'publisher_chapter_evidence': {'sha256': 'a' * 64}},
|
||||
'tags': [{'namespace': PUBLISHER + ':format', 'value': 'chapter'}]}
|
||||
return {'doi': doi, 'retained_paper_id': paper, 'retained_programme_record_id': record,
|
||||
'retained_submission_id': attempt['id'], 'retained_programme_decision_id': attempt['decisions'][0]['id'],
|
||||
'move_from_edition_id': original, 'source_row': row, 'before_packet_sha256': packet_digest(packet)}
|
||||
|
||||
def run_plan(self, entries, *database):
|
||||
path = self.root / 'plan.json'
|
||||
raw = json.dumps({'status': 'readonly_same_submission_reconciliation_plan', 'database_written': False,
|
||||
'plans': entries, 'guarded_same_submission_plans': len(entries)}).encode()
|
||||
path.write_bytes(raw)
|
||||
main(['maintain', 'eccv-publication-reconciliation', *(database or ('--db', str(self.db))),
|
||||
'--plan', str(path), '--plan-sha256', sha256(raw).hexdigest(), '--springer-previous-count', '1'])
|
||||
|
||||
def test_preserves_history_and_updates_all_three_coverage_scopes(self):
|
||||
entry = self.entries[0]
|
||||
before = entity_packet(self.conn, entry['retained_paper_id'])
|
||||
self.run_plan([entry])
|
||||
packet = entity_packet(self.conn, entry['retained_paper_id'])
|
||||
self.assertEqual(packet['paper'], before['paper'])
|
||||
programme, other = packet['attempts']
|
||||
self.assertEqual((programme['id'], programme['edition_id']), (entry['retained_submission_id'], self.target))
|
||||
self.assertEqual(other, before['attempts'][1])
|
||||
self.assertEqual(programme['decisions'][0], before['attempts'][0]['decisions'][0])
|
||||
self.assertEqual([row['status'] for row in programme['decisions']], ['accepted', 'accepted'])
|
||||
self.assertEqual({r['source'] for r in programme['sources']}, {PROGRAMME, PUBLISHER})
|
||||
self.assertEqual(len(packet['source_records']), 3)
|
||||
self.assertEqual(resolve(self.conn, entry['retained_programme_record_id'], '1',
|
||||
'ECCV', 2026, 'program_unclassified', None), self.target)
|
||||
coverage = {(edition, source): (status, expected, imported)
|
||||
for edition, source, status, expected, imported in self.conn.execute('''SELECT
|
||||
e.track,s.name,c.status,c.expected_count,c.imported_count FROM edition_sources c
|
||||
JOIN editions e ON e.id=c.edition_id JOIN sources s ON s.id=c.source_id''')}
|
||||
self.assertEqual(coverage, {('program_unclassified', PROGRAMME): ('partial', 2834, 2),
|
||||
('main', PROGRAMME): ('partial', None, 1),
|
||||
('main', PUBLISHER): ('partial', 2835, 2)})
|
||||
with self.assertRaisesRegex(ValueError, 'already applied'):
|
||||
self.run_plan([entry])
|
||||
|
||||
def test_changed_history_rolls_back_earlier_observation(self):
|
||||
first, second = self.entries[:2]
|
||||
stale = {**second, 'before_packet_sha256': 'b' * 64}
|
||||
packet = entity_packet(self.conn, first['retained_paper_id'])
|
||||
with self.assertRaisesRegex(ValueError, 'Reviewed paper history changed'):
|
||||
self.run_plan([first, stale])
|
||||
self.assertEqual(entity_packet(self.conn, first['retained_paper_id']), packet)
|
||||
self.assertIsNone(self.conn.execute(
|
||||
"SELECT 1 FROM identifiers WHERE scheme='doi' AND value=?", (first['doi'],)).fetchone())
|
||||
self.assertEqual(self.conn.execute('SELECT count(*) FROM edition_sources').fetchone()[0], 2)
|
||||
|
||||
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the remote PostgreSQL test service')
|
||||
def test_postgres_write_verifies_history_from_the_read_backend(self):
|
||||
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
|
||||
schema = 'pa_eccv_' + uuid.uuid4().hex[:12]
|
||||
|
||||
def cleanup():
|
||||
with connect(service) as conn:
|
||||
for name, in conn.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
|
||||
conn.execute('DROP SCHEMA ' + quote(name) + ' CASCADE')
|
||||
|
||||
self.addCleanup(cleanup)
|
||||
self.conn.close()
|
||||
with redirect_stdout(io.StringIO()):
|
||||
migrate(self.db, service, self.root / 'migration.json', schema=schema, min_free_gib=0)
|
||||
entry = self.entries[0]
|
||||
with closing(ReadConnection(service, schema)) as conn:
|
||||
before = entity_packet(conn, entry['retained_paper_id'])
|
||||
reviewed = {**entry, 'before_packet_sha256': packet_digest(before)}
|
||||
self.run_plan([reviewed], '--pg-service', service, '--pg-schema', schema)
|
||||
with closing(ReadConnection(service, schema)) as conn:
|
||||
packet = entity_packet(conn, entry['retained_paper_id'])
|
||||
self.assertEqual(packet['attempts'][1], before['attempts'][1])
|
||||
self.assertEqual(packet['attempts'][0]['id'], entry['retained_submission_id'])
|
||||
self.assertEqual(packet['attempts'][0]['edition_id'], self.target)
|
||||
self.assertEqual(len(packet['attempts'][0]['decisions']), 2)
|
||||
@@ -13,7 +13,7 @@ import uuid
|
||||
|
||||
from paper_atlas.db.backend import open_store
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
from paper_atlas.db.source_revisions import available, require_row, require_manifest
|
||||
from paper_atlas.db.source_revisions import available, decode_payload, require_row, require_manifest
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
|
||||
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||
@@ -69,6 +69,28 @@ class RevisionCases:
|
||||
def plan(self, c):
|
||||
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
|
||||
|
||||
def test_large_payload_round_trips_and_protects_next_revision(self):
|
||||
for key in ('previous_rows', 'rows'):
|
||||
self.inventory[key][-1]['metadata']['evidence'] = '来源原文' * 100000
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c)
|
||||
applied = apply_plan(c, plan)
|
||||
c.commit()
|
||||
saved = c.execute('SELECT records_json,plan_json FROM source_scope_revisions').fetchone()
|
||||
expected = (self.inventory['rows'], plan)
|
||||
for i, value in enumerate(expected):
|
||||
self.assertTrue(saved[i].startswith('gzip+base64:v1:'))
|
||||
self.assertLess(len(saved[i]), len(json.dumps(value, ensure_ascii=False).encode()) // 10)
|
||||
self.assertEqual(decode_payload(saved[i]), value)
|
||||
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
|
||||
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||
require_row(c, sid, self.inventory['rows'][-1], {'list_url': URL}, {})
|
||||
next_inventory = copy.deepcopy(self.inventory)
|
||||
next_inventory['previous_rows'] = copy.deepcopy(self.inventory['rows'])
|
||||
next_inventory['rows'][-1]['metadata']['reviewed'] = True
|
||||
next_plan = proposal(c, next_inventory, database_identity(c, self.target.get('db')))
|
||||
self.assertEqual(next_plan['before']['parent_sha256'], applied['version_sha256'])
|
||||
|
||||
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
|
||||
|
||||
@@ -9,7 +9,7 @@ import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors.springer_selected import (
|
||||
BOOK, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
|
||||
BOOK, SCOPE, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
|
||||
)
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
@@ -153,6 +153,28 @@ class SpringerSelectedTests(unittest.TestCase):
|
||||
html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"')
|
||||
self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract'])
|
||||
|
||||
def test_single_name_preserves_literal_and_requires_ordered_display_confirmation(self):
|
||||
html = f"""
|
||||
<meta name="citation_doi" content="{BOOK}_1">
|
||||
<meta name="citation_inbook_title" content="{SCOPE['book_title']}">
|
||||
<meta name="citation_publication_date" content="{SCOPE['publication_year']}">
|
||||
<meta name="citation_title" content="Example">
|
||||
<meta name="citation_author" content="Rajkumar">
|
||||
<meta name="citation_author" content="Doe, Jane">
|
||||
<meta name="citation_firstpage" content="1">
|
||||
<meta name="citation_lastpage" content="5">
|
||||
"""
|
||||
article = {'@type':'ScholarlyArticle', 'author':[{'name':'Rajkumar'}, {'name':'Jane Doe'}]}
|
||||
script = '<script type="application/ld+json">{}</script>'
|
||||
chapter = parse_chapter(html+script.format(json.dumps(article)), BOOK+'_1')
|
||||
self.assertEqual(chapter['authors'], ['Rajkumar', {'family':'Doe', 'given':'Jane'}])
|
||||
self.assertEqual(chapter['authors_citation_raw'], ['Rajkumar','Doe, Jane'])
|
||||
with self.assertRaisesRegex(ValueError, 'matching Springer display author list'):
|
||||
parse_chapter(html, BOOK+'_1')
|
||||
article['author'].reverse()
|
||||
with self.assertRaisesRegex(ValueError, 'matching Springer display author list'):
|
||||
parse_chapter(html+script.format(json.dumps(article)), BOOK+'_1')
|
||||
|
||||
def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self):
|
||||
html = self.fixture['toc']['springer-toc1.html']
|
||||
self.assertEqual(len(parse_toc(html)), 17)
|
||||
|
||||
Reference in New Issue
Block a user